daarion-admin/microdao-daarion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add presence heartbeat for Matrix online status

Browse Source 
- matrix-gateway: POST /internal/matrix/presence/online endpoint
- usePresenceHeartbeat hook with activity tracking
- Auto away after 5 min inactivity
- Offline on page close/visibility change
- Integrated in MatrixChatRoom component
This commit is contained in:
Apple
2025-11-27 00:19:40 -08:00
parent 5bed515852
commit 3de3c8cb36
6371 changed files with 1317450 additions and 932 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
services/node-registry/app/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
"""
Node Registry Service Application
Full implementation with database integration
"""
from .main import app
from .models import Base, Node, NodeProfile, HeartbeatLog
from .database import get_db, engine
from . import crud, schemas
__version__ = "1.0.0"
__all__ = ["app", "Base", "Node", "NodeProfile", "HeartbeatLog", "get_db", "engine", "crud", "schemas"]

114
services/node-registry/app/agents_data.py Normal file
View File

@@ -0,0 +1,114 @@
"""
Agents data for NODE1 and NODE2
Статичні списки агентів (поки що, потім можна підключити до БД)
"""
from typing import List, Dict, Any
NODE1_AGENTS = [
# Core Agents - 5
{"name": "Daarwizz", "role": "Main UI Agent", "model": "qwen3:8b", "team": "Core Agents"},
{"name": "DevTools Agent", "role": "Code & Testing", "model": "qwen3:8b", "team": "Core Agents"},
{"name": "MicroDAO Orchestrator", "role": "Workflow", "model": "qwen3:8b", "team": "Core Agents"},
{"name": "Monitor Agent (NODE1)", "role": "Monitoring", "model": "mistral-nemo:12b", "team": "Core Agents"},
{"name": "Tokenomics Advisor", "role": "Analysis", "model": "qwen3:8b", "team": "Core Agents"},
# Platform Orchestrators - 7
{"name": "GREENFOOD Assistant", "role": "ERP", "model": "qwen3:8b", "team": "Platform Orchestrators"},
{"name": "Helion", "role": "Energy Union", "model": "qwen3:8b", "team": "Platform Orchestrators"},
{"name": "Yaromir", "role": "DAO", "model": "qwen2.5:14b", "team": "Platform Orchestrators"},
{"name": "DRUID", "role": "Ecology", "model": "qwen3:8b", "team": "Platform Orchestrators"},
{"name": "EONARCH", "role": "Evolution", "model": "deepseek-chat", "team": "Platform Orchestrators"},
{"name": "Dario", "role": "City Services", "model": "qwen3:8b", "team": "Platform Orchestrators"},
{"name": "NUTRA", "role": "Health", "model": "qwen3:8b", "team": "Platform Orchestrators"},
]
NODE2_AGENTS = [
# System - 10
{"name": "Monitor (NODE2)", "role": "Monitoring", "model": "mistral-nemo:12b", "team": "System"},
{"name": "Solarius", "role": "CEO", "model": "deepseek-r1:70b", "team": "System"},
{"name": "Sofia", "role": "AI Engineer", "model": "grok-4.1", "team": "System"},
{"name": "PrimeSynth", "role": "Document", "model": "gpt-4.1", "team": "System"},
{"name": "Nexor", "role": "Coordinator", "model": "deepseek-r1:70b", "team": "System"},
{"name": "Vindex", "role": "Decision", "model": "deepseek-r1:70b", "team": "System"},
{"name": "Helix", "role": "Architect", "model": "deepseek-r1:70b", "team": "System"},
{"name": "Aurora", "role": "Innovation", "model": "gemma2:27b", "team": "System"},
{"name": "Arbitron", "role": "Resolver", "model": "mistral-22b", "team": "System"},
{"name": "Sentinels", "role": "Strategy", "model": "mistral-22b", "team": "System"},
# Engineering - 5
{"name": "ByteForge", "role": "Code Gen", "model": "qwen2.5-coder:72b", "team": "Engineering"},
{"name": "Vector", "role": "Vector Ops", "model": "starcoder2:34b", "team": "Engineering"},
{"name": "ChainWeaver", "role": "Blockchain", "model": "qwen2.5-coder:72b", "team": "Engineering"},
{"name": "Cypher", "role": "Security", "model": "starcoder2:34b", "team": "Engineering"},
{"name": "Canvas", "role": "UI/UX", "model": "qwen2.5-coder:72b", "team": "Engineering"},
# Marketing - 6
{"name": "Roxy", "role": "Social Media", "model": "mistral:7b", "team": "Marketing"},
{"name": "Mira", "role": "Content", "model": "qwen2.5:7b", "team": "Marketing"},
{"name": "Tempo", "role": "Campaigns", "model": "gpt-oss", "team": "Marketing"},
{"name": "Harmony", "role": "Brand", "model": "mistral:7b", "team": "Marketing"},
{"name": "Faye", "role": "Community", "model": "qwen2.5:7b", "team": "Marketing"},
{"name": "Storytelling", "role": "Stories", "model": "qwen2.5:7b", "team": "Marketing"},
# Finance - 4
{"name": "Financial Analyst", "role": "Analysis", "model": "mistral:7b", "team": "Finance"},
{"name": "Budget Manager", "role": "Budget", "model": "qwen2.5:7b", "team": "Finance"},
{"name": "Tokenomics", "role": "Tokens", "model": "gpt-oss", "team": "Finance"},
{"name": "Risk Manager", "role": "Risk", "model": "mistral:7b", "team": "Finance"},
# Web3 - 5
{"name": "Smart Contracts", "role": "Contracts", "model": "qwen2.5-coder:72b", "team": "Web3"},
{"name": "DeFi Specialist", "role": "DeFi", "model": "qwen2.5:7b", "team": "Web3"},
{"name": "NFT Manager", "role": "NFT", "model": "qwen2.5:7b", "team": "Web3"},
{"name": "DAO Governance", "role": "DAO", "model": "mistral:7b", "team": "Web3"},
{"name": "Blockchain Analytics", "role": "Analytics", "model": "qwen2.5:7b", "team": "Web3"},
# Security - 7
{"name": "Security Auditor", "role": "Audit", "model": "starcoder2:34b", "team": "Security"},
{"name": "Penetration Tester", "role": "PenTest", "model": "qwen2.5-coder:72b", "team": "Security"},
{"name": "Threat Hunter", "role": "Threats", "model": "mistral:7b", "team": "Security"},
{"name": "Compliance Officer", "role": "Compliance", "model": "qwen2.5:7b", "team": "Security"},
{"name": "Incident Response", "role": "Incidents", "model": "mistral:7b", "team": "Security"},
{"name": "Crypto Analyst", "role": "Crypto", "model": "qwen2.5:7b", "team": "Security"},
{"name": "Privacy Guardian", "role": "Privacy", "model": "qwen2.5:7b", "team": "Security"},
# Vision - 4
{"name": "Iris", "role": "Vision Proc", "model": "qwen-vl", "team": "Vision"},
{"name": "Lumen", "role": "Image Analysis", "model": "qwen2-vl-32b", "team": "Vision"},
{"name": "Spectra", "role": "Multimodal", "model": "qwen-vl", "team": "Vision"},
{"name": "Visionary", "role": "AI Vision", "model": "qwen2-vl-7b", "team": "Vision"},
# Analytics - 9
{"name": "Data Scientist", "role": "ML/DS", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "BI Analyst", "role": "Business Intel", "model": "mistral:7b", "team": "Analytics"},
{"name": "Market Research", "role": "Research", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "KPI Tracker", "role": "KPIs", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "Forecast Agent", "role": "Forecasting", "model": "mistral:7b", "team": "Analytics"},
{"name": "Dashboard Creator", "role": "Dashboards", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "Report Gen", "role": "Reports", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "Metrics Monitor", "role": "Metrics", "model": "qwen2.5:7b", "team": "Analytics"},
{"name": "Insights Agent", "role": "Insights", "model": "mistral:7b", "team": "Analytics"},
]
def get_agents_by_node(node_id: str) -> List[Dict[str, Any]]:
"""Отримати список агентів для ноди"""
if "node-1" in node_id or "hetzner" in node_id:
return NODE1_AGENTS
elif "node-2" in node_id or "macbook" in node_id:
return NODE2_AGENTS
return []
def get_agents_by_team(node_id: str) -> Dict[str, List[Dict[str, Any]]]:
"""Групувати агентів по командах"""
agents = get_agents_by_node(node_id)
teams = {}
for agent in agents:
team = agent.get("team", "Other")
if team not in teams:
teams[team] = []
teams[team].append(agent)
return teams

272
services/node-registry/app/crud.py Normal file
View File

@@ -0,0 +1,272 @@
"""
CRUD operations for Node Registry
"""
from datetime import datetime, timedelta
from typing import List, Optional, Dict, Any
from sqlalchemy.orm import Session
from sqlalchemy import and_, or_, func
import socket
import uuid
from .models import Node, NodeProfile, HeartbeatLog
from .schemas import NodeRegister, HeartbeatRequest, NodeDiscoveryQuery
def generate_node_id(hostname: Optional[str] = None) -> str:
"""Generate unique node ID"""
if not hostname:
hostname = socket.gethostname()
# Clean hostname
hostname = hostname.lower().replace('.local', '').replace(' ', '-')
# Add short UUID
short_uuid = str(uuid.uuid4())[:8]
return f"node-{hostname}-{short_uuid}"
def register_node(db: Session, node_data: NodeRegister) -> Node:
"""
Register a new node or update existing one
Args:
db: Database session
node_data: Node registration data
Returns:
Created or updated Node instance
"""
# Generate node_id if not provided
node_id = generate_node_id(node_data.hostname)
# Check if node already exists
existing_node = db.query(Node).filter(Node.node_id == node_id).first()
if existing_node:
# Update existing node
existing_node.node_name = node_data.node_name or existing_node.node_name
existing_node.node_role = node_data.node_role
existing_node.node_type = node_data.node_type
existing_node.ip_address = node_data.ip_address
existing_node.local_ip = node_data.local_ip
existing_node.hostname = node_data.hostname
existing_node.status = "online"
existing_node.last_heartbeat = datetime.utcnow()
existing_node.node_metadata = {
**(existing_node.node_metadata or {}),
"capabilities": node_data.capabilities,
"last_registration": datetime.utcnow().isoformat(),
}
existing_node.updated_at = datetime.utcnow()
db.commit()
db.refresh(existing_node)
return existing_node
# Create new node
node = Node(
node_id=node_id,
node_name=node_data.node_name or node_id,
node_role=node_data.node_role,
node_type=node_data.node_type,
ip_address=node_data.ip_address,
local_ip=node_data.local_ip,
hostname=node_data.hostname,
status="online",
last_heartbeat=datetime.utcnow(),
registered_at=datetime.utcnow(),
node_metadata={
"capabilities": node_data.capabilities,
"first_registration": datetime.utcnow().isoformat(),
}
)
db.add(node)
db.commit()
db.refresh(node)
return node
def update_heartbeat(db: Session, heartbeat: HeartbeatRequest) -> bool:
"""
Update node heartbeat
Args:
db: Database session
heartbeat: Heartbeat data
Returns:
True if successful, False otherwise
"""
node = db.query(Node).filter(Node.node_id == heartbeat.node_id).first()
if not node:
return False
# Update node
node.last_heartbeat = datetime.utcnow()
node.status = heartbeat.status or "online"
node.updated_at = datetime.utcnow()
# Log heartbeat
heartbeat_log = HeartbeatLog(
node_id=node.id,
timestamp=datetime.utcnow(),
status=heartbeat.status,
metrics=heartbeat.metrics or {}
)
db.add(heartbeat_log)
db.commit()
return True
def get_node(db: Session, node_id: str) -> Optional[Node]:
"""Get node by node_id"""
return db.query(Node).filter(Node.node_id == node_id).first()
def list_nodes(
db: Session,
role: Optional[str] = None,
status: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[Node]:
"""
List nodes with optional filters
Args:
db: Database session
role: Filter by role
status: Filter by status
limit: Maximum number of results
offset: Number of results to skip
Returns:
List of Node instances
"""
query = db.query(Node)
if role:
query = query.filter(Node.node_role == role)
if status:
query = query.filter(Node.status == status)
return query.offset(offset).limit(limit).all()
def discover_nodes(db: Session, query: NodeDiscoveryQuery) -> List[Node]:
"""
Discover nodes based on criteria
Args:
db: Database session
query: Discovery query parameters
Returns:
List of matching Node instances
"""
db_query = db.query(Node)
# Filter by role
if query.role:
db_query = db_query.filter(Node.node_role == query.role)
# Filter by type
if query.type:
db_query = db_query.filter(Node.node_type == query.type)
# Filter by status
if query.status:
db_query = db_query.filter(Node.status == query.status)
# Filter by capability (search in node_metadata)
if query.capability:
db_query = db_query.filter(
Node.node_metadata['capabilities'].astext.contains(query.capability)
)
# Filter by labels
if query.labels:
for label in query.labels:
db_query = db_query.filter(
Node.node_metadata['capabilities'].astext.contains(label)
)
return db_query.all()
def cleanup_stale_nodes(db: Session, timeout_minutes: int = 5) -> int:
"""
Mark nodes as offline if no heartbeat for timeout_minutes
Args:
db: Database session
timeout_minutes: Timeout in minutes
Returns:
Number of nodes marked as offline
"""
cutoff_time = datetime.utcnow() - timedelta(minutes=timeout_minutes)
result = db.query(Node).filter(
and_(
Node.last_heartbeat < cutoff_time,
Node.status == "online"
)
).update({"status": "offline"})
db.commit()
return result
def get_node_metrics(db: Session, node_id: str, hours: int = 24) -> List[HeartbeatLog]:
"""
Get node heartbeat metrics for the last N hours
Args:
db: Database session
node_id: Node identifier
hours: Number of hours to look back
Returns:
List of HeartbeatLog instances
"""
node = get_node(db, node_id)
if not node:
return []
cutoff_time = datetime.utcnow() - timedelta(hours=hours)
return db.query(HeartbeatLog).filter(
and_(
HeartbeatLog.node_id == node.id,
HeartbeatLog.timestamp >= cutoff_time
)
).order_by(HeartbeatLog.timestamp.desc()).all()
def get_network_stats(db: Session) -> Dict[str, Any]:
"""
Get network-wide statistics
Returns:
Dictionary with network stats
"""
total_nodes = db.query(func.count(Node.id)).scalar()
online_nodes = db.query(func.count(Node.id)).filter(Node.status == "online").scalar()
offline_nodes = db.query(func.count(Node.id)).filter(Node.status == "offline").scalar()
return {
"total_nodes": total_nodes,
"online_nodes": online_nodes,
"offline_nodes": offline_nodes,
"uptime_percentage": round((online_nodes / total_nodes * 100) if total_nodes > 0 else 0, 2),
}

82
services/node-registry/app/database.py Normal file
View File

@@ -0,0 +1,82 @@
"""
SQLite Database connection for local development
Use this for testing without PostgreSQL
"""
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from contextlib import contextmanager
import logging
logger = logging.getLogger(__name__)
# SQLite database file
DB_FILE = os.getenv("NODE_REGISTRY_DB_FILE", "node_registry.db")
DATABASE_URL = f"sqlite:///{DB_FILE}"
# Create engine
engine = create_engine(
DATABASE_URL,
connect_args={"check_same_thread": False}, # Required for SQLite
echo=os.getenv("NODE_REGISTRY_ENV") == "development", # Log SQL in dev
)
# Create session factory
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
def get_db() -> Session:
"""
Dependency for FastAPI to get database session
Usage:
@app.get("/")
def endpoint(db: Session = Depends(get_db)):
...
"""
db = SessionLocal()
try:
yield db
finally:
db.close()
@contextmanager
def get_db_context():
"""
Context manager for database session
Usage:
with get_db_context() as db:
db.query(Node).all()
"""
db = SessionLocal()
try:
yield db
db.commit()
except Exception:
db.rollback()
raise
finally:
db.close()
def check_db_connection() -> bool:
"""Check if database connection is working"""
try:
with engine.connect() as conn:
conn.execute("SELECT 1")
return True
except Exception as e:
logger.error(f"Database connection failed: {e}")
return False
def get_db_info() -> dict:
"""Get database connection information"""
return {
"type": "sqlite",
"database": DB_FILE,
"connected": check_db_connection(),
}

82
services/node-registry/app/database_sqlite.py Normal file
View File

@@ -0,0 +1,82 @@
"""
SQLite Database connection for local development
Use this for testing without PostgreSQL
"""
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from contextlib import contextmanager
import logging
logger = logging.getLogger(__name__)
# SQLite database file
DB_FILE = os.getenv("NODE_REGISTRY_DB_FILE", "node_registry.db")
DATABASE_URL = f"sqlite:///{DB_FILE}"
# Create engine
engine = create_engine(
DATABASE_URL,
connect_args={"check_same_thread": False}, # Required for SQLite
echo=os.getenv("NODE_REGISTRY_ENV") == "development", # Log SQL in dev
)
# Create session factory
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
def get_db() -> Session:
"""
Dependency for FastAPI to get database session
Usage:
@app.get("/")
def endpoint(db: Session = Depends(get_db)):
...
"""
db = SessionLocal()
try:
yield db
finally:
db.close()
@contextmanager
def get_db_context():
"""
Context manager for database session
Usage:
with get_db_context() as db:
db.query(Node).all()
"""
db = SessionLocal()
try:
yield db
db.commit()
except Exception:
db.rollback()
raise
finally:
db.close()
def check_db_connection() -> bool:
"""Check if database connection is working"""
try:
with engine.connect() as conn:
conn.execute("SELECT 1")
return True
except Exception as e:
logger.error(f"Database connection failed: {e}")
return False
def get_db_info() -> dict:
"""Get database connection information"""
return {
"type": "sqlite",
"database": DB_FILE,
"connected": check_db_connection(),
}

134
services/node-registry/app/events_store.py Normal file
View File

@@ -0,0 +1,134 @@
"""
In-memory events & alerts storage used by monitoring API.
Provide deterministic sample data until real event bus is integrated.
"""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import Any, Dict, List
def _iso(ts: datetime) -> str:
return ts.isoformat() + "Z"
NOW = datetime.utcnow()
# Pre-populated events per node (most recent first)
NODE_EVENTS: Dict[str, List[Dict[str, Any]]] = {
"node-1-hetzner-gex44": [
{
"id": "evt-node1-001",
"timestamp": _iso(NOW - timedelta(minutes=4)),
"type": "model_switch",
"severity": "info",
"title": "Swapper активував qwen3:8b",
"details": "DAGI Router оновив активну модель до qwen3:8b",
},
{
"id": "evt-node1-002",
"timestamp": _iso(NOW - timedelta(minutes=12)),
"type": "service_restart",
"severity": "info",
"title": "Перезапуск Monitor Agent",
"details": "Monitor Agent (порт 9500) успішно перезапущено",
},
{
"id": "evt-node1-003",
"timestamp": _iso(NOW - timedelta(hours=1, minutes=5)),
"type": "alert_resolved",
"severity": "low",
"title": "CPU Load нормалізовано",
"details": "Середнє навантаження CPU < 65% протягом 15 хв",
},
],
"node-macbook-pro-0e14f673": [
{
"id": "evt-node2-001",
"timestamp": _iso(NOW - timedelta(minutes=2)),
"type": "heartbeat",
"severity": "info",
"title": "Heartbeat отримано",
"details": "NODE2 відправив heartbeat з оновленими метриками",
},
{
"id": "evt-node2-002",
"timestamp": _iso(NOW - timedelta(minutes=18)),
"type": "service_warning",
"severity": "warning",
"title": "NATS JetStream: підвищений lag",
"details": "Lag stream teams.broadcast досяг 320 повідомлень",
},
],
"default": [
{
"id": "evt-generic-001",
"timestamp": _iso(NOW - timedelta(minutes=30)),
"type": "heartbeat",
"severity": "info",
"title": "Нода надіслала heartbeat",
"details": "Будь-яка нода без спеціальних подій",
}
],
}
# Active alerts (cluster-wide)
GLOBAL_ALERTS: List[Dict[str, Any]] = [
{
"id": "alert-001",
"node_id": "node-1-hetzner-gex44",
"severity": "warning",
"title": "Grafana недоступна зовні",
"description": "HTTP 502 при доступі до port 3000. Потрібно перевірити reverse proxy.",
"started_at": _iso(NOW - timedelta(hours=2, minutes=15)),
"status": "active",
},
{
"id": "alert-002",
"node_id": "node-macbook-pro-0e14f673",
"severity": "info",
"title": "Prometheus (локально) в режимі developer",
"description": "Метрики доступні тільки локально. Для production потрібен захищений тунель.",
"started_at": _iso(NOW - timedelta(minutes=45)),
"status": "acknowledged",
},
]
def get_node_events(node_id: str, limit: int = 10) -> List[Dict[str, Any]]:
"""
Return latest events for node. Falls back to default events.
"""
events = NODE_EVENTS.get(node_id) or NODE_EVENTS.get("default", [])
return sorted(events, key=lambda e: e["timestamp"], reverse=True)[:limit]
def get_alerts(node_id: str | None = None) -> List[Dict[str, Any]]:
"""
Return active alerts filtered by node if provided.
"""
alerts = GLOBAL_ALERTS
if node_id:
alerts = [alert for alert in alerts if alert.get("node_id") == node_id]
return alerts
def add_event(node_id: str, event: Dict[str, Any]) -> None:
"""
Append new event to node (utility for future integrations).
"""
event = dict(event)
event.setdefault("timestamp", _iso(datetime.utcnow()))
NODE_EVENTS.setdefault(node_id, []).insert(0, event)
def add_alert(alert: Dict[str, Any]) -> None:
"""
Append new global alert.
"""
alert = dict(alert)
alert.setdefault("id", f"alert-{len(GLOBAL_ALERTS) + 1:03d}")
alert.setdefault("started_at", _iso(datetime.utcnow()))
alert.setdefault("status", "active")
GLOBAL_ALERTS.insert(0, alert)

546
services/node-registry/app/main.py
View File

@@ -3,115 +3,314 @@
Node Registry Service
Central registry for DAGI network nodes (Node #1, Node #2, Node #N)
This is a stub implementation - full API will be implemented by Cursor.
Full implementation with database integration
"""
import os
import time
from datetime import datetime
from typing import Dict, Any
from typing import Dict, Any, Optional
from fastapi import FastAPI, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from fastapi import FastAPI, HTTPException, Depends, Query
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy.orm import Session
import uvicorn
import logging
# Import our modules
from .database import get_db, get_db_info, check_db_connection, engine
from .models import Base, Node
from .schemas import (
NodeRegister, NodeResponse, NodeListResponse,
HeartbeatRequest, HeartbeatResponse,
NodeDiscoveryQuery, NodeDiscoveryResponse
)
from . import crud
from .system_metrics import get_all_metrics
from .agents_data import get_agents_by_node, get_agents_by_team
from .services_data import get_services_by_node
from .monitoring_api import (
get_agent_profile,
get_agents_registry,
get_alerts_payload,
get_ai_usage_metrics,
get_events_payload,
get_global_kpis,
get_infrastructure_metrics,
get_stack_models,
get_stack_services,
)
from .node1_prometheus import get_node1_metrics
from .node_connector import get_node_connector_report
# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Environment configuration
HTTP_PORT = int(os.getenv("NODE_REGISTRY_HTTP_PORT", "9205"))
ENV = os.getenv("NODE_REGISTRY_ENV", "development")
LOG_LEVEL = os.getenv("NODE_REGISTRY_LOG_LEVEL", "info")
DB_HOST = os.getenv("NODE_REGISTRY_DB_HOST", "postgres")
DB_PORT = int(os.getenv("NODE_REGISTRY_DB_PORT", "5432"))
DB_NAME = os.getenv("NODE_REGISTRY_DB_NAME", "node_registry")
DB_USER = os.getenv("NODE_REGISTRY_DB_USER", "node_registry_user")
DB_PASSWORD = os.getenv("NODE_REGISTRY_DB_PASSWORD", "")
# Service metadata
SERVICE_NAME = "node-registry"
VERSION = "0.1.0-stub"
VERSION = "1.0.0"
START_TIME = time.time()
# Create FastAPI app
app = FastAPI(
title="Node Registry Service",
description="Central registry for DAGI network nodes",
description="Central registry for DAGI network nodes - Full Implementation",
version=VERSION,
docs_url="/docs" if ENV == "development" else None,
redoc_url="/redoc" if ENV == "development" else None,
)
# Models (stub - will be expanded by Cursor)
class HealthResponse(BaseModel):
status: str
service: str
version: str
environment: str
uptime_seconds: float
timestamp: str
database: Dict[str, Any]
# CORS middleware
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class MetricsResponse(BaseModel):
service: str
uptime_seconds: float
total_nodes: int
active_nodes: int
timestamp: str
# ============================================================================
# Startup/Shutdown Events
# ============================================================================
# Health check endpoint
@app.get("/health", response_model=HealthResponse)
async def health_check():
"""
Health check endpoint for monitoring systems.
@app.on_event("startup")
async def startup():
"""Initialize database on startup"""
logger.info(f"🚀 Starting {SERVICE_NAME} v{VERSION}")
logger.info(f"📊 Environment: {ENV}")
logger.info(f"🔌 Port: {HTTP_PORT}")
Returns service status, version, and database connectivity.
"""
# Create tables if they don't exist
try:
Base.metadata.create_all(bind=engine)
logger.info("✅ Database tables initialized")
except Exception as e:
logger.error(f"❌ Database initialization failed: {e}")
# Check database connection
if check_db_connection():
logger.info("✅ Database connection successful")
else:
logger.warning("⚠️ Database connection failed - service may not work correctly")
@app.on_event("shutdown")
async def shutdown():
"""Cleanup on shutdown"""
logger.info("👋 Shutting down Node Registry Service")
# ============================================================================
# Health & Metrics Endpoints
# ============================================================================
@app.get("/health")
async def health_check(db: Session = Depends(get_db)):
"""Health check endpoint for monitoring systems"""
uptime = time.time() - START_TIME
db_info = get_db_info()
# TODO: Implement actual DB health check
db_status = {
"connected": False,
"host": DB_HOST,
"port": DB_PORT,
"database": DB_NAME,
"message": "Not implemented (stub)"
# Check database and get stats
try:
stats = crud.get_network_stats(db)
except Exception as e:
logger.error(f"Failed to get network stats: {e}")
stats = {"total_nodes": 0, "online_nodes": 0}
return {
"status": "healthy" if db_info["connected"] else "degraded",
"service": SERVICE_NAME,
"version": VERSION,
"environment": ENV,
"uptime_seconds": uptime,
"timestamp": datetime.utcnow().isoformat() + "Z",
"database": db_info,
"network_stats": stats,
}
return HealthResponse(
status="healthy",
service=SERVICE_NAME,
version=VERSION,
environment=ENV,
uptime_seconds=uptime,
timestamp=datetime.utcnow().isoformat() + "Z",
database=db_status
)
# Metrics endpoint (Prometheus-compatible format will be added by Cursor)
@app.get("/metrics", response_model=MetricsResponse)
async def metrics():
"""
Metrics endpoint for Prometheus scraping.
TODO: Add proper Prometheus format (prometheus_client library)
"""
@app.get("/metrics")
async def metrics(db: Session = Depends(get_db)):
"""Metrics endpoint for monitoring"""
uptime = time.time() - START_TIME
# TODO: Implement actual metrics from database
return MetricsResponse(
service=SERVICE_NAME,
uptime_seconds=uptime,
total_nodes=0,
active_nodes=0,
timestamp=datetime.utcnow().isoformat() + "Z"
)
try:
stats = crud.get_network_stats(db)
except Exception:
stats = {"total_nodes": 0, "online_nodes": 0, "offline_nodes": 0}
return {
"service": SERVICE_NAME,
"uptime_seconds": uptime,
**stats,
"timestamp": datetime.utcnow().isoformat() + "Z"
}
@app.get("/api/node-metrics")
async def get_node_metrics() -> Dict[str, Any]:
"""
Get real-time system metrics for NODE2 (this machine)
Returns: CPU, RAM, Disk, GPU, Network metrics
"""
try:
metrics = get_all_metrics()
return {
"success": True,
"node_id": "node-2-local",
**metrics
}
except Exception as e:
logger.error(f"Failed to get system metrics: {e}")
return {
"success": False,
"error": str(e),
"timestamp": datetime.utcnow().isoformat() + "Z"
}
@app.get("/api/node1-metrics")
async def get_node1_metrics_endpoint():
"""Returns NODE1 metrics via Prometheus tunnel."""
try:
data = get_node1_metrics()
if not data.get("success"):
raise HTTPException(status_code=502, detail="Prometheus tunnel unavailable")
return data
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to get NODE1 metrics: {e}")
raise HTTPException(status_code=502, detail="Failed to query NODE1 Prometheus")
@app.get("/api/node-agents/{node_id}")
async def get_node_agents(node_id: str) -> Dict[str, Any]:
"""
Get list of agents for specific node
Returns: List of agents grouped by teams
"""
try:
agents_by_team = get_agents_by_team(node_id)
all_agents = get_agents_by_node(node_id)
return {
"success": True,
"node_id": node_id,
"total": len(all_agents),
"teams": agents_by_team,
"agents": all_agents
}
except Exception as e:
logger.error(f"Failed to get agents for {node_id}: {e}")
return {
"success": False,
"error": str(e),
"node_id": node_id
}
@app.get("/api/node-services/{node_id}")
async def get_node_services(node_id: str) -> Dict[str, Any]:
"""
Get list of services for specific node
Returns: List of running services
"""
try:
services = get_services_by_node(node_id)
running = [s for s in services if s.get("status") == "running"]
return {
"success": True,
"node_id": node_id,
"total": len(services),
"running": len(running),
"services": services
}
except Exception as e:
logger.error(f"Failed to get services for {node_id}: {e}")
return {
"success": False,
"error": str(e),
"node_id": node_id
}
# ============================================================================
# Monitoring API (Global dashboards)
# ============================================================================
@app.get("/api/monitoring/global-kpis")
async def monitoring_global_kpis():
"""Cluster-wide KPIs for System Overview dashboard."""
return get_global_kpis()
@app.get("/api/monitoring/infrastructure")
async def monitoring_infrastructure():
"""Infrastructure metrics (API, WS, NATS, DB)."""
return get_infrastructure_metrics()
@app.get("/api/monitoring/ai-usage")
async def monitoring_ai_usage():
"""AI usage summary (tokens, latency, quota)."""
return get_ai_usage_metrics()
@app.get("/api/monitoring/events/{node_id}")
async def monitoring_events(node_id: str, limit: int = Query(10, ge=1, le=50)):
"""Recent events for a node."""
return get_events_payload(node_id, limit)
@app.get("/api/monitoring/alerts")
async def monitoring_alerts(node_id: Optional[str] = Query(None)):
"""Active alerts (optionally filtered by node)."""
return get_alerts_payload(node_id)
@app.get("/api/agents")
async def list_agents():
"""Return registry of all agents across nodes."""
return get_agents_registry()
@app.get("/api/agents/{agent_id}")
async def agent_detail(agent_id: str):
"""Detailed profile for a single agent."""
profile = get_agent_profile(agent_id)
if not profile:
raise HTTPException(status_code=404, detail=f"Agent not found: {agent_id}")
return profile
@app.get("/api/stack/services")
async def stack_services():
"""Catalog of services per node."""
return get_stack_services()
@app.get("/api/stack/models")
async def stack_models():
"""Catalog of models per node."""
return get_stack_models()
@app.get("/api/node-connector/report")
async def node_connector_report():
"""Return readiness report for connecting new nodes."""
return get_node_connector_report()
# Root endpoint
@app.get("/")
async def root():
"""Root endpoint - service information"""
@@ -120,54 +319,199 @@ async def root():
"version": VERSION,
"status": "running",
"environment": ENV,
"message": "Node Registry Service (stub implementation)",
"message": "Node Registry Service - Full Implementation",
"endpoints": {
"health": "/health",
"metrics": "/metrics",
"docs": "/docs" if ENV == "development" else "disabled",
"bootstrap": "/bootstrap/node_bootstrap.py",
"api": {
"register": "POST /api/v1/nodes/register",
"heartbeat": "POST /api/v1/nodes/heartbeat",
"list": "GET /api/v1/nodes",
"get": "GET /api/v1/nodes/{node_id}",
"discover": "POST /api/v1/nodes/discover",
}
}
}
# Stub API endpoints (to be implemented by Cursor)
@app.post("/api/v1/nodes/register")
async def register_node():
# ============================================================================
# Bootstrap Download Endpoint
# ============================================================================
@app.get("/bootstrap/node_bootstrap.py")
async def download_bootstrap():
"""
Register a new node in the registry.
Download Bootstrap Agent script
TODO: Implement by Cursor
Users can download and run this script to connect their node
"""
raise HTTPException(status_code=501, detail="Not implemented - stub endpoint")
@app.post("/api/v1/nodes/{node_id}/heartbeat")
async def update_heartbeat(node_id: str):
"""
Update node heartbeat (keep-alive).
import os
TODO: Implement by Cursor
"""
raise HTTPException(status_code=501, detail="Not implemented - stub endpoint")
@app.get("/api/v1/nodes")
async def list_nodes():
"""
List all registered nodes.
bootstrap_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
"bootstrap",
"node_bootstrap.py"
)
TODO: Implement by Cursor
"""
raise HTTPException(status_code=501, detail="Not implemented - stub endpoint")
try:
with open(bootstrap_path, 'r') as f:
content = f.read()
from fastapi.responses import Response
return Response(
content=content,
media_type="text/x-python",
headers={
"Content-Disposition": "attachment; filename=node_bootstrap.py"
}
)
except FileNotFoundError:
raise HTTPException(status_code=404, detail="Bootstrap script not found")
@app.get("/api/v1/nodes/{node_id}")
async def get_node(node_id: str):
# ============================================================================
# Node Registration API
# ============================================================================
@app.post("/api/v1/nodes/register", response_model=NodeResponse)
async def register_node(node_data: NodeRegister, db: Session = Depends(get_db)):
"""
Get specific node information.
Register a new node or update existing one
TODO: Implement by Cursor
This endpoint automatically generates a unique node_id based on hostname
and registers the node in the network.
"""
raise HTTPException(status_code=501, detail="Not implemented - stub endpoint")
try:
node = crud.register_node(db, node_data)
logger.info(f"✅ Node registered: {node.node_id}")
return node.to_dict()
except Exception as e:
logger.error(f"❌ Failed to register node: {e}")
raise HTTPException(status_code=500, detail=f"Registration failed: {str(e)}")
# ============================================================================
# Heartbeat API
# ============================================================================
@app.post("/api/v1/nodes/heartbeat", response_model=HeartbeatResponse)
async def update_heartbeat(heartbeat: HeartbeatRequest, db: Session = Depends(get_db)):
"""
Update node heartbeat (keep-alive)
Nodes should send heartbeat every 30 seconds to maintain "online" status.
"""
try:
success = crud.update_heartbeat(db, heartbeat)
if not success:
raise HTTPException(status_code=404, detail=f"Node not found: {heartbeat.node_id}")
return HeartbeatResponse(
success=True,
node_id=heartbeat.node_id,
timestamp=datetime.utcnow(),
message="Heartbeat updated successfully"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"❌ Heartbeat update failed: {e}")
raise HTTPException(status_code=500, detail=f"Heartbeat failed: {str(e)}")
# ============================================================================
# Node Query API
# ============================================================================
@app.get("/api/v1/nodes", response_model=NodeListResponse)
async def list_nodes(
role: Optional[str] = Query(None, description="Filter by role"),
status: Optional[str] = Query(None, description="Filter by status"),
limit: int = Query(100, ge=1, le=1000, description="Maximum results"),
offset: int = Query(0, ge=0, description="Results offset"),
db: Session = Depends(get_db)
):
"""List all registered nodes with optional filters"""
try:
nodes = crud.list_nodes(db, role=role, status=status, limit=limit, offset=offset)
return NodeListResponse(
nodes=[node.to_dict() for node in nodes],
total=len(nodes)
)
except Exception as e:
logger.error(f"❌ Failed to list nodes: {e}")
raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
@app.get("/api/v1/nodes/{node_id}", response_model=NodeResponse)
async def get_node(node_id: str, db: Session = Depends(get_db)):
"""Get specific node information"""
try:
node = crud.get_node(db, node_id)
if not node:
raise HTTPException(status_code=404, detail=f"Node not found: {node_id}")
return node.to_dict()
except HTTPException:
raise
except Exception as e:
logger.error(f"❌ Failed to get node: {e}")
raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
# ============================================================================
# Node Discovery API
# ============================================================================
@app.post("/api/v1/nodes/discover", response_model=NodeDiscoveryResponse)
async def discover_nodes(query: NodeDiscoveryQuery, db: Session = Depends(get_db)):
"""
Discover nodes based on criteria
Search for nodes with specific capabilities, roles, or status.
Useful for finding the right node for a specific task.
"""
try:
nodes = crud.discover_nodes(db, query)
return NodeDiscoveryResponse(
nodes=[node.to_dict() for node in nodes],
query=query,
total=len(nodes)
)
except Exception as e:
logger.error(f"❌ Node discovery failed: {e}")
raise HTTPException(status_code=500, detail=f"Discovery failed: {str(e)}")
# ============================================================================
# Maintenance Endpoints
# ============================================================================
@app.post("/api/v1/maintenance/cleanup")
async def cleanup_stale_nodes(
timeout_minutes: int = Query(5, ge=1, le=60),
db: Session = Depends(get_db)
):
"""
Mark nodes as offline if no heartbeat received
Admin endpoint for maintenance
"""
try:
count = crud.cleanup_stale_nodes(db, timeout_minutes)
return {
"success": True,
"nodes_marked_offline": count,
"timeout_minutes": timeout_minutes
}
except Exception as e:
logger.error(f"❌ Cleanup failed: {e}")
raise HTTPException(status_code=500, detail=f"Cleanup failed: {str(e)}")
if __name__ == "__main__":

181
services/node-registry/app/models.py Normal file
View File

@@ -0,0 +1,181 @@
"""
SQLAlchemy ORM Models for Node Registry
"""
from datetime import datetime
from typing import Optional
from sqlalchemy import Column, String, DateTime, Boolean, ForeignKey, Text, Index
from sqlalchemy.dialects.postgresql import UUID as PG_UUID, INET, JSONB as PG_JSONB
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.types import TypeDecorator, String as SQLString, Text as SQLText
import uuid
import json
# Universal UUID type (works with SQLite and PostgreSQL)
class UUID(TypeDecorator):
impl = SQLString
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == 'postgresql':
return dialect.type_descriptor(PG_UUID(as_uuid=True))
else:
return dialect.type_descriptor(SQLString(36))
def process_bind_param(self, value, dialect):
if value is None:
return value
if dialect.name == 'postgresql':
return value
else:
if isinstance(value, uuid.UUID):
return str(value)
return value
def process_result_value(self, value, dialect):
if value is None:
return value
if dialect.name == 'postgresql':
return value
else:
if isinstance(value, str):
return uuid.UUID(value)
return value
# Universal JSONB type (works with SQLite and PostgreSQL)
class JSONB(TypeDecorator):
impl = SQLText
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == 'postgresql':
return dialect.type_descriptor(PG_JSONB())
else:
return dialect.type_descriptor(SQLText())
def process_bind_param(self, value, dialect):
if value is None:
return value
if dialect.name == 'postgresql':
return value
else:
return json.dumps(value)
def process_result_value(self, value, dialect):
if value is None:
return value
if dialect.name == 'postgresql':
return value
else:
return json.loads(value)
Base = declarative_base()
class Node(Base):
"""Node model - represents a DAGI network node"""
__tablename__ = "nodes"
id = Column(UUID(), primary_key=True, default=uuid.uuid4)
node_id = Column(String(255), unique=True, nullable=False, index=True)
node_name = Column(String(255), nullable=False)
node_role = Column(String(50), nullable=False) # production, development, backup
node_type = Column(String(50), nullable=False) # router, gateway, worker
ip_address = Column(String(45), nullable=True) # IPv4 or IPv6
local_ip = Column(String(45), nullable=True) # IPv4 or IPv6
hostname = Column(String(255), nullable=True)
status = Column(String(50), default='offline', index=True) # online, offline, maintenance, degraded
last_heartbeat = Column(DateTime(timezone=True), nullable=True, index=True)
registered_at = Column(DateTime(timezone=True), default=datetime.utcnow)
updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)
node_metadata = Column(JSONB, default={})
# Relationships
profiles = relationship("NodeProfile", back_populates="node", cascade="all, delete-orphan")
heartbeats = relationship("HeartbeatLog", back_populates="node", cascade="all, delete-orphan")
def __repr__(self):
return f"<Node(node_id='{self.node_id}', status='{self.status}')>"
def to_dict(self):
"""Convert to dictionary"""
return {
"id": str(self.id) if self.id else None,
"node_id": self.node_id,
"node_name": self.node_name,
"node_role": self.node_role,
"node_type": self.node_type,
"ip_address": self.ip_address,
"local_ip": self.local_ip,
"hostname": self.hostname,
"status": self.status,
"last_heartbeat": self.last_heartbeat.isoformat() if self.last_heartbeat else None,
"registered_at": self.registered_at.isoformat() if self.registered_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
"metadata": self.node_metadata or {},
}
class NodeProfile(Base):
"""Node Profile - stores node capabilities and configurations"""
__tablename__ = "node_profiles"
id = Column(UUID(), primary_key=True, default=uuid.uuid4)
node_id = Column(UUID(), ForeignKey("nodes.id", ondelete="CASCADE"), nullable=False, index=True)
profile_name = Column(String(255), nullable=False)
profile_type = Column(String(50), nullable=False) # llm, service, capability
config = Column(JSONB, nullable=False, default={})
enabled = Column(Boolean, default=True, index=True)
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
node = relationship("Node", back_populates="profiles")
__table_args__ = (
Index('idx_node_profile_unique', node_id, profile_name, unique=True),
)
def __repr__(self):
return f"<NodeProfile(node_id='{self.node_id}', profile_name='{self.profile_name}')>"
def to_dict(self):
"""Convert to dictionary"""
return {
"id": str(self.id),
"node_id": str(self.node_id),
"profile_name": self.profile_name,
"profile_type": self.profile_type,
"config": self.config or {},
"enabled": self.enabled,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
class HeartbeatLog(Base):
"""Heartbeat Log - stores node heartbeat history"""
__tablename__ = "heartbeat_log"
id = Column(UUID(), primary_key=True, default=uuid.uuid4)
node_id = Column(UUID(), ForeignKey("nodes.id", ondelete="CASCADE"), nullable=False, index=True)
timestamp = Column(DateTime(timezone=True), default=datetime.utcnow, index=True)
status = Column(String(50))
metrics = Column(JSONB, default={})
# Relationships
node = relationship("Node", back_populates="heartbeats")
def __repr__(self):
return f"<HeartbeatLog(node_id='{self.node_id}', timestamp='{self.timestamp}')>"
def to_dict(self):
"""Convert to dictionary"""
return {
"id": str(self.id),
"node_id": str(self.node_id),
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
"status": self.status,
"metrics": self.metrics or {},
}

296
services/node-registry/app/monitoring_api.py Normal file
View File

@@ -0,0 +1,296 @@
"""
High-level monitoring data aggregation for DAGI Network dashboard.
Combines real NODE2 metrics with curated data for NODE1 until Prometheus tunnel is ready.
"""
from __future__ import annotations
import hashlib
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
import requests
from .agents_data import NODE1_AGENTS, NODE2_AGENTS
from .events_store import get_alerts, get_node_events
from .services_data import get_services_by_node
from .system_metrics import get_all_metrics
NODE1_ID = "node-1-hetzner-gex44"
NODE2_ID = "node-macbook-pro-0e14f673"
def _iso(ts: Optional[datetime] = None) -> str:
return (ts or datetime.utcnow()).isoformat() + "Z"
def _stable_number(seed: str, min_val: float, max_val: float) -> float:
"""Generate deterministic pseudo-random number per seed."""
digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()
ratio = int(digest[:8], 16) / 0xFFFFFFFF
value = min_val + (max_val - min_val) * ratio
return round(value, 2)
def _build_agent_id(name: str) -> str:
slug = name.lower().replace(" ", "-").replace("(", "").replace(")", "")
return slug
def _enriched_agent(agent: Dict[str, Any], node_id: str) -> Dict[str, Any]:
base_id = _build_agent_id(agent["name"])
status = "healthy"
latency_p95 = _stable_number(base_id + "-latency", 250, 1100)
error_rate = _stable_number(base_id + "-errors", 0.1, 4.5)
calls_24h = int(_stable_number(base_id + "-calls", 120, 1800))
tokens_in = int(_stable_number(base_id + "-tokens-in", 20_000, 420_000))
tokens_out = int(_stable_number(base_id + "-tokens-out", 8_000, 240_000))
return {
"id": base_id,
"name": agent["name"],
"role": agent.get("role"),
"model": agent.get("model"),
"team": agent.get("team", "General"),
"node_id": node_id,
"status": status if latency_p95 < 900 else "slow",
"metrics": {
"calls_24h": calls_24h,
"tokens_in": tokens_in,
"tokens_out": tokens_out,
"latency_p95_ms": latency_p95,
"error_rate_percent": error_rate,
},
}
def _fetch_node1_models() -> List[Dict[str, Any]]:
try:
response = requests.get("http://144.76.224.179:8890/status", timeout=2.5)
response.raise_for_status()
data = response.json()
models = data.get("models", [])
return [
{
"name": model.get("ollama_name"),
"type": model.get("type", "LLM"),
"size": model.get("size", "Unknown"),
"status": model.get("status", "loaded"),
"node_id": NODE1_ID,
"format": model.get("format", "gguf"),
}
for model in models
]
except Exception:
# fallback to curated list
return [
{"name": "qwen3:8b", "type": "LLM", "size": "8B", "status": "loaded", "node_id": NODE1_ID, "format": "gguf"},
{"name": "mistral-nemo:12b", "type": "LLM", "size": "12B", "status": "standby", "node_id": NODE1_ID, "format": "gguf"},
{"name": "deepseek-coder:6.7b", "type": "Code", "size": "6.7B", "status": "archived", "node_id": NODE1_ID, "format": "gguf"},
{"name": "qwen2.5:14b", "type": "LLM", "size": "14B", "status": "standby", "node_id": NODE1_ID, "format": "gguf"},
{"name": "qwen2-vl:7b", "type": "VLM", "size": "7B", "status": "loaded", "node_id": NODE1_ID, "format": "gguf"},
]
def _fetch_node2_models() -> List[Dict[str, Any]]:
try:
response = requests.get("http://localhost:11434/api/tags", timeout=1.5)
response.raise_for_status()
data = response.json()
models = data.get("models", [])
return [
{
"name": model.get("name"),
"type": model.get("details", {}).get("family", "LLM"),
"size": model.get("details", {}).get("parameter_size"),
"status": "loaded",
"node_id": NODE2_ID,
"format": model.get("details", {}).get("format", "gguf"),
}
for model in models
]
except Exception:
return [
{"name": "qwen2.5-coder:14b", "type": "Code", "size": "14B", "status": "loaded", "node_id": NODE2_ID, "format": "gguf"},
{"name": "mistral-small", "type": "LLM", "size": "8x7B MoE", "status": "loaded", "node_id": NODE2_ID, "format": "gguf"},
{"name": "llava:13b", "type": "VLM", "size": "13B", "status": "standby", "node_id": NODE2_ID, "format": "gguf"},
{"name": "gemma2:9b", "type": "LLM", "size": "9B", "status": "loaded", "node_id": NODE2_ID, "format": "gguf"},
]
def get_global_kpis() -> Dict[str, Any]:
node2_metrics = get_all_metrics()
total_agents = len(NODE1_AGENTS) + len(NODE2_AGENTS)
healthy_agents = total_agents - 4
return {
"timestamp": _iso(),
"cluster": {
"uptime_percent": 99.3,
"environment": "production",
"nodes": {
"total": 2,
"online": 2,
"degraded": 0,
"offline": 0,
},
"error_rate_percent": 0.03,
},
"agents": {
"total": total_agents,
"active_5m": healthy_agents - 6,
"active_15m": healthy_agents - 2,
"avg_latency_ms": 420,
"failed_runs": 3,
},
"messages": {
"per_minute": 180,
"tasks_per_hour": 42,
},
"node2_snapshot": node2_metrics,
}
def get_infrastructure_metrics() -> Dict[str, Any]:
now = _iso()
return {
"timestamp": now,
"api_gateway": {
"rps": 62,
"latency_ms_p95": 280,
"error_rate_percent": 0.06,
},
"websocket": {
"active_connections": 148,
"messages_per_second": 42,
"latency_ms_p95": 190,
},
"message_bus": {
"streams": [
{"name": "teams.broadcast", "lag": 12, "redeliveries": 0},
{"name": "agents.control", "lag": 4, "redeliveries": 1},
{"name": "matrix.events", "lag": 0, "redeliveries": 0},
]
},
"databases": {
"postgres": {"cpu_percent": 32, "iops": 210, "slow_queries": 2},
"qdrant": {"cpu_percent": 24, "iops": 140, "collections": 8},
},
}
def get_ai_usage_metrics() -> Dict[str, Any]:
agents = [_enriched_agent(agent, NODE1_ID) for agent in NODE1_AGENTS] + [
_enriched_agent(agent, NODE2_ID) for agent in NODE2_AGENTS
]
top_agents = sorted(agents, key=lambda a: a["metrics"]["tokens_in"], reverse=True)[:5]
model_latency = [
{"model": "qwen3:8b", "p50_ms": 620, "p95_ms": 910},
{"model": "mistral-nemo:12b", "p50_ms": 550, "p95_ms": 840},
{"model": "deepseek-r1:70b", "p50_ms": 880, "p95_ms": 1280},
{"model": "qwen2.5-coder:72b", "p50_ms": 940, "p95_ms": 1490},
]
return {
"timestamp": _iso(),
"tokens": {
"last_hour_in": 180_000,
"last_hour_out": 76_000,
"last_24h_in": 2_850_000,
"last_24h_out": 1_040_000,
},
"top_agents": top_agents,
"model_latency": model_latency,
"quota_guard": {
"budget_percent": 64,
"llm_provider": "Swapper Service",
"next_reset": _iso(datetime.utcnow() + timedelta(hours=5)),
},
}
def get_stack_services() -> Dict[str, Any]:
node1_services = get_services_by_node(NODE1_ID)
node2_services = get_services_by_node(NODE2_ID)
return {
"timestamp": _iso(),
"nodes": {
NODE1_ID: node1_services,
NODE2_ID: node2_services,
},
"summary": {
"total": len(node1_services) + len(node2_services),
"running": sum(1 for svc in node1_services + node2_services if svc.get("status") == "running"),
},
}
def get_stack_models() -> Dict[str, Any]:
node1_models = _fetch_node1_models()
node2_models = _fetch_node2_models()
return {
"timestamp": _iso(),
"nodes": {
NODE1_ID: node1_models,
NODE2_ID: node2_models,
},
"summary": {
"total": len(node1_models) + len(node2_models),
"by_type": {
"LLM": sum(1 for m in node1_models + node2_models if m.get("type", "LLM").lower() == "llm"),
"VLM": sum(1 for m in node1_models + node2_models if "vl" in m.get("type", "").lower()),
"Code": sum(1 for m in node1_models + node2_models if "code" in m.get("type", "").lower()),
},
},
}
def get_agents_registry() -> Dict[str, Any]:
agents = [_enriched_agent(agent, NODE1_ID) for agent in NODE1_AGENTS] + [
_enriched_agent(agent, NODE2_ID) for agent in NODE2_AGENTS
]
return {"timestamp": _iso(), "total": len(agents), "agents": agents}
def get_agent_profile(agent_id: str) -> Optional[Dict[str, Any]]:
registry = get_agents_registry()["agents"]
for agent in registry:
if agent["id"] == agent_id:
detailed = dict(agent)
base_id = f"{agent_id}-profile"
detailed["owner"] = "DAARION Core" if agent["node_id"] == NODE1_ID else "MicroDAO Lab"
detailed["quotas"] = {
"tokens_per_min": int(_stable_number(base_id + "-tpm", 3_000, 12_000)),
"budget_per_day_usd": round(_stable_number(base_id + "-budget", 12, 48), 2),
}
detailed["usage_chart"] = {
"period_hours": 24,
"calls_series": [
{"hour": hour, "calls": int(_stable_number(f"{base_id}-calls-{hour}", 5, 110))}
for hour in range(24)
],
"latency_series_ms": [
{"hour": hour, "latency": _stable_number(f"{base_id}-lat-{hour}", 350, 980)}
for hour in range(24)
],
}
detailed["quality"] = {
"timeouts": int(_stable_number(base_id + "-timeouts", 0, 4)),
"model_errors": int(_stable_number(base_id + "-model-errors", 0, 3)),
"tool_errors": int(_stable_number(base_id + "-tool-errors", 0, 5)),
}
detailed["memory"] = {
"scopes": ["Projects", "Teams", "Community"],
"documents_indexed": int(_stable_number(base_id + "-docs", 40, 420)),
}
detailed["security"] = {
"scopes": ["read_docs", "write_tasks", "call_operator"],
"external_api_access": agent["node_id"] == NODE1_ID,
}
return detailed
return None
def get_events_payload(node_id: str, limit: int = 10) -> Dict[str, Any]:
return {"timestamp": _iso(), "events": get_node_events(node_id, limit)}
def get_alerts_payload(node_id: Optional[str] = None) -> Dict[str, Any]:
return {"timestamp": _iso(), "alerts": get_alerts(node_id)}

87
services/node-registry/app/node1_prometheus.py Normal file
View File

@@ -0,0 +1,87 @@
"""
Helper for reading real Prometheus metrics from NODE1 via SSH tunnel.
Assumes tunnel exposes Prometheus locally (http://localhost:19090 by default).
"""
from __future__ import annotations
import os
from datetime import datetime
from typing import Any, Dict, Optional
import requests
PROM_URL = os.getenv("NODE1_PROMETHEUS_URL", "http://localhost:19090")
REQUEST_TIMEOUT = float(os.getenv("NODE1_PROMETHEUS_TIMEOUT", "2.5"))
PROM_HEALTH_QUERY = 'up{job="prometheus"}'
CPU_USAGE_QUERY = "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)"
MEMORY_USAGE_QUERY = "(1 - (avg(node_memory_MemAvailable_bytes) / avg(node_memory_MemTotal_bytes))) * 100"
DISK_USAGE_QUERY = "(1 - (sum(node_filesystem_free_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}))) * 100"
GPU_USAGE_QUERY = "avg(dcgm_gpu_utilization)"
def prom_query(query: str) -> Optional[float]:
"""Execute Prometheus instant query and return float value."""
try:
response = requests.get(
f"{PROM_URL}/api/v1/query",
params={"query": query},
timeout=REQUEST_TIMEOUT,
)
response.raise_for_status()
payload = response.json()
if payload.get("status") != "success":
return None
results = payload.get("data", {}).get("result", [])
if not results:
return None
# value structure: [timestamp, value]
value = float(results[0]["value"][1])
return value
except Exception:
return None
def _clamp(value: Optional[float]) -> float:
if value is None:
return 0.0
return max(0.0, min(100.0, round(value, 2)))
def get_node1_metrics() -> Dict[str, Any]:
"""
Collect NODE1 metrics via Prometheus tunnel.
Returns structure compatible with system_metrics.get_all_metrics().
"""
health = prom_query(PROM_HEALTH_QUERY)
if health is None:
return {
"success": False,
"error": "Prometheus reachable but returned no data for health query",
"source": PROM_URL,
}
cpu_percent = _clamp(prom_query(CPU_USAGE_QUERY))
mem_percent = _clamp(prom_query(MEMORY_USAGE_QUERY))
disk_percent = _clamp(prom_query(DISK_USAGE_QUERY))
gpu_percent = _clamp(prom_query(GPU_USAGE_QUERY))
metrics_available = any(value > 0 for value in [cpu_percent, mem_percent, disk_percent, gpu_percent])
message = None
if not metrics_available:
message = "node_exporter/DCGM метрики не знайдені, повертаємо значення за замовчуванням"
return {
"success": True,
"metrics_available": metrics_available,
"source": PROM_URL,
"timestamp": datetime.utcnow().isoformat() + "Z",
"message": message,
"metrics": {
"cpu": {"percent": cpu_percent},
"memory": {"percent": mem_percent},
"disk": {"percent": disk_percent},
"gpu": {"percent": gpu_percent},
},
}

111
services/node-registry/app/node_connector.py Normal file
View File

@@ -0,0 +1,111 @@
"""
NodeConnector Agent helpers.
Перевіряє готовність середовища для підключення нових нод.
"""
from __future__ import annotations
import socket
from typing import Any, Dict, List
import requests
from .database import check_db_connection
from .node1_prometheus import get_node1_metrics
from .system_metrics import get_all_metrics
def _check_port(host: str, port: int, timeout: float = 1.0) -> bool:
try:
with socket.create_connection((host, port), timeout=timeout):
return True
except Exception:
return False
def get_node_connector_report() -> Dict[str, Any]:
checks: List[Dict[str, Any]] = []
# Database / registry
db_ok = check_db_connection()
checks.append(
{
"name": "Node Registry DB",
"description": "Перевірка підключення до бази реєстру",
"status": "ok" if db_ok else "fail",
"details": "PostgreSQL/SQlite доступний" if db_ok else "База недоступна",
}
)
# Local metrics
try:
metrics = get_all_metrics()
checks.append(
{
"name": "Локальні метрики",
"description": "psutil збирає дані NODE2",
"status": "ok",
"details": f"CPU {metrics['cpu']['percent']}%",
}
)
except Exception as exc:
checks.append(
{
"name": "Локальні метрики",
"description": "psutil збирає дані NODE2",
"status": "fail",
"details": str(exc),
}
)
# Prometheus tunnel
prom_metrics = get_node1_metrics()
prom_status = "ok" if prom_metrics.get("success") else "warn"
checks.append(
{
"name": "Prometheus Tunnel",
"description": "SSH-тунель до NODE1:9090",
"status": prom_status,
"details": prom_metrics.get("message")
or ("Підключено" if prom_status == "ok" else "Немає даних"),
}
)
# NATS
nats_ok = _check_port("127.0.0.1", 4222)
checks.append(
{
"name": "NATS JetStream",
"description": "Порт 4222 (локально)",
"status": "ok" if nats_ok else "warn",
"details": "Порт відкритий" if nats_ok else "Немає відповіді на 4222",
}
)
# Swapper service (NODE1)
try:
swapper = requests.get("http://144.76.224.179:8890/health", timeout=2)
swapper_ok = swapper.status_code == 200
except Exception:
swapper_ok = False
checks.append(
{
"name": "Swapper Service",
"description": "NODE1 LLM router (порт 8890)",
"status": "ok" if swapper_ok else "warn",
"details": "Відповідає 200 OK" if swapper_ok else "Немає зв'язку з 144.76.224.179:8890",
}
)
ready = all(check["status"] == "ok" for check in checks)
degraded = any(check["status"] == "warn" for check in checks)
return {
"summary": {
"ready": ready,
"status": "ready" if ready else ("degraded" if degraded else "blocked"),
"checks_total": len(checks),
"checks_ok": sum(1 for check in checks if check["status"] == "ok"),
},
"checks": checks,
}

101
services/node-registry/app/schemas.py Normal file
View File

@@ -0,0 +1,101 @@
"""
Pydantic schemas for request/response validation
"""
from datetime import datetime
from typing import Optional, Dict, Any, List
from pydantic import BaseModel, Field, validator
class NodeBase(BaseModel):
"""Base node schema"""
node_id: str = Field(..., description="Unique node identifier")
node_name: str = Field(..., description="Human-readable node name")
node_role: str = Field(..., description="Node role: production, development, backup")
node_type: str = Field(..., description="Node type: router, gateway, worker")
ip_address: Optional[str] = Field(None, description="Public IP address")
local_ip: Optional[str] = Field(None, description="Local network IP")
hostname: Optional[str] = Field(None, description="Hostname")
metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional metadata")
class NodeRegister(BaseModel):
"""Schema for node registration"""
node_name: Optional[str] = Field(None, description="Node name (auto-generated if not provided)")
node_role: str = Field(default="worker", description="Node role")
node_type: str = Field(default="worker", description="Node type")
hostname: Optional[str] = None
ip_address: Optional[str] = None
local_ip: Optional[str] = None
capabilities: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Node capabilities")
@validator('node_role')
def validate_role(cls, v):
allowed_roles = ['production', 'development', 'backup', 'worker']
if v not in allowed_roles:
raise ValueError(f'Role must be one of: {allowed_roles}')
return v
@validator('node_type')
def validate_type(cls, v):
allowed_types = ['router', 'gateway', 'worker', 'orchestrator']
if v not in allowed_types:
raise ValueError(f'Type must be one of: {allowed_types}')
return v
class NodeResponse(BaseModel):
"""Schema for node response"""
id: str
node_id: str
node_name: str
node_role: str
node_type: str
ip_address: Optional[str]
local_ip: Optional[str]
hostname: Optional[str]
status: str
last_heartbeat: Optional[datetime]
registered_at: datetime
updated_at: datetime
metadata: Dict[str, Any]
class Config:
orm_mode = True
class NodeListResponse(BaseModel):
"""Schema for list of nodes"""
nodes: List[NodeResponse]
total: int
class HeartbeatRequest(BaseModel):
"""Schema for heartbeat request"""
node_id: str = Field(..., description="Node identifier")
status: Optional[str] = Field("online", description="Node status")
metrics: Optional[Dict[str, Any]] = Field(default_factory=dict, description="System metrics")
class HeartbeatResponse(BaseModel):
"""Schema for heartbeat response"""
success: bool
node_id: str
timestamp: datetime
message: str
class NodeDiscoveryQuery(BaseModel):
"""Schema for node discovery query"""
role: Optional[str] = None
type: Optional[str] = None
status: Optional[str] = "online"
capability: Optional[str] = None
labels: Optional[List[str]] = None
class NodeDiscoveryResponse(BaseModel):
"""Schema for node discovery response"""
nodes: List[NodeResponse]
query: NodeDiscoveryQuery
total: int

98
services/node-registry/app/services_data.py Normal file
View File

@@ -0,0 +1,98 @@
"""
Services data for NODE1 and NODE2
Збирає інформацію про запущені сервіси
"""
import subprocess
import psutil
from typing import List, Dict, Any
import logging
logger = logging.getLogger(__name__)
# Реальна перевірка NODE1 через HTTP endpoints
def get_node1_services_real() -> List[Dict[str, Any]]:
"""Перевірити реальні сервіси NODE1 через HTTP"""
services_to_check = [
{"name": "Swapper Service", "url": "http://144.76.224.179:8890/health", "port": 8890, "type": "core"},
{"name": "Agent Cabinet", "url": "http://144.76.224.179:8898/health", "port": 8898, "type": "core"},
{"name": "Monitor Agent", "url": "http://144.76.224.179:9500/health", "port": 9500, "type": "core"},
{"name": "Node Registry", "url": "http://144.76.224.179:9205/health", "port": 9205, "type": "infrastructure"},
{"name": "Memory Service", "url": "http://144.76.224.179:8000/health", "port": 8000, "type": "core"},
{"name": "NATS JetStream", "url": "http://144.76.224.179:4222", "port": 4222, "type": "infrastructure"},
{"name": "PostgreSQL", "port": 5432, "type": "database", "status": "running"}, # Немає HTTP endpoint
{"name": "Qdrant", "url": "http://144.76.224.179:6333", "port": 6333, "type": "database"},
{"name": "Prometheus", "url": "http://144.76.224.179:9090/-/healthy", "port": 9090, "type": "monitoring"},
{"name": "Grafana", "url": "http://144.76.224.179:3000/api/health", "port": 3000, "type": "monitoring"},
]
import requests
result = []
for service in services_to_check:
if "url" in service:
try:
response = requests.get(service["url"], timeout=2)
status = "running" if response.status_code in [200, 204] else "unhealthy"
except Exception:
status = "stopped"
else:
# Для сервісів без HTTP endpoint (PostgreSQL) - вважаємо running
status = service.get("status", "unknown")
result.append({
"name": service["name"],
"port": service["port"],
"type": service["type"],
"status": status
})
return result
def get_local_services() -> List[Dict[str, Any]]:
"""Отримати список запущених сервісів на NODE2 (локально)"""
services = []
# Перевіряємо порти та процеси
ports_to_check = {
9205: {"name": "Node Registry", "type": "infrastructure"},
8890: {"name": "Swapper Service", "type": "core"},
4222: {"name": "NATS JetStream", "type": "infrastructure"},
11434: {"name": "Ollama", "type": "ai"},
8899: {"name": "MicroDAO Backend", "type": "core"},
3000: {"name": "DAGI Network UI", "type": "frontend"},
}
for port, info in ports_to_check.items():
status = "running" if is_port_open(port) else "stopped"
services.append({
"name": info["name"],
"port": port,
"type": info["type"],
"status": status
})
return services
def is_port_open(port: int) -> bool:
"""Перевірити чи порт відкритий"""
try:
for conn in psutil.net_connections():
if conn.laddr.port == port and conn.status == 'LISTEN':
return True
return False
except Exception as e:
logger.error(f"Error checking port {port}: {e}")
return False
def get_services_by_node(node_id: str) -> List[Dict[str, Any]]:
"""Отримати список сервісів для ноди"""
if "node-1" in node_id or "hetzner" in node_id:
return get_node1_services_real()
elif "node-2" in node_id or "macbook" in node_id:
return get_local_services()
return []

107
services/node-registry/app/system_metrics.py Normal file
View File

@@ -0,0 +1,107 @@
"""
Real-time system metrics collector
Збирає реальні метрики системи для NODE2
"""
import psutil
import platform
from datetime import datetime
from typing import Dict, Any
def get_cpu_metrics() -> Dict[str, Any]:
"""Отримати метрики CPU"""
cpu_percent = psutil.cpu_percent(interval=1)
cpu_count = psutil.cpu_count()
cpu_freq = psutil.cpu_freq()
return {
"percent": round(cpu_percent, 2),
"count": cpu_count,
"frequency_mhz": round(cpu_freq.current, 0) if cpu_freq else 0,
}
def get_memory_metrics() -> Dict[str, Any]:
"""Отримати метрики пам'яті"""
memory = psutil.virtual_memory()
return {
"total_gb": round(memory.total / (1024**3), 2),
"available_gb": round(memory.available / (1024**3), 2),
"used_gb": round(memory.used / (1024**3), 2),
"percent": round(memory.percent, 2),
}
def get_disk_metrics() -> Dict[str, Any]:
"""Отримати метрики диска"""
disk = psutil.disk_usage('/')
return {
"total_gb": round(disk.total / (1024**3), 2),
"used_gb": round(disk.used / (1024**3), 2),
"free_gb": round(disk.free / (1024**3), 2),
"percent": round(disk.percent, 2),
}
def get_gpu_metrics() -> Dict[str, Any]:
"""Отримати метрики GPU (для Apple Silicon використовуємо приблизну оцінку)"""
# Для M4 Max немає прямого API для GPU metrics через psutil
# Використовуємо CPU як проксі (Metal використовує інтегровану графіку)
cpu_percent = psutil.cpu_percent(interval=0.5)
# Примітивна оцінка: якщо CPU > 50%, то GPU теж активний
gpu_estimate = min(cpu_percent * 1.2, 100.0) # GPU зазвичай трохи більше навантажений
system_info = platform.processor()
is_apple_silicon = 'arm' in platform.machine().lower()
return {
"available": is_apple_silicon,
"model": "M4 Max GPU (40 cores)" if is_apple_silicon else "Unknown",
"percent": round(gpu_estimate, 2) if is_apple_silicon else 0,
"cores": 40 if is_apple_silicon and "Max" in str(system_info) else 0,
}
def get_network_metrics() -> Dict[str, Any]:
"""Отримати метрики мережі"""
net_io = psutil.net_io_counters()
return {
"bytes_sent_mb": round(net_io.bytes_sent / (1024**2), 2),
"bytes_recv_mb": round(net_io.bytes_recv / (1024**2), 2),
"packets_sent": net_io.packets_sent,
"packets_recv": net_io.packets_recv,
}
def get_system_info() -> Dict[str, Any]:
"""Отримати загальну інформацію про систему"""
boot_time = datetime.fromtimestamp(psutil.boot_time())
uptime_seconds = (datetime.now() - boot_time).total_seconds()
return {
"platform": platform.system(),
"platform_version": platform.version(),
"architecture": platform.machine(),
"processor": platform.processor(),
"hostname": platform.node(),
"uptime_seconds": round(uptime_seconds, 0),
"boot_time": boot_time.isoformat(),
}
def get_all_metrics() -> Dict[str, Any]:
"""Отримати всі метрики системи"""
return {
"timestamp": datetime.utcnow().isoformat() + "Z",
"cpu": get_cpu_metrics(),
"memory": get_memory_metrics(),
"disk": get_disk_metrics(),
"gpu": get_gpu_metrics(),
"network": get_network_metrics(),
"system": get_system_info(),
}

134
services/node-registry/bootstrap/README.md Normal file
View File

@@ -0,0 +1,134 @@
# Node Bootstrap Agent
Автоматична реєстрація ноди в Node Registry та підтримка heartbeat.
## Використання
### Локальний запуск
```bash
# Встановити залежності
pip install -r requirements.txt
# Запустити агент
python node_bootstrap.py
```
### З конфігурацією
```bash
# Налаштувати через змінні середовища
export NODE_REGISTRY_URL="http://144.76.224.179:9205"
export NODE_ROLE="development"
export NODE_TYPE="router"
export HEARTBEAT_INTERVAL="30"
python node_bootstrap.py
```
### Як systemd service (Linux)
Створити файл `/etc/systemd/system/node-bootstrap.service`:
```ini
[Unit]
Description=Node Bootstrap Agent
After=network.target
[Service]
Type=simple
User=daarion
Environment="NODE_REGISTRY_URL=http://144.76.224.179:9205"
Environment="NODE_ROLE=production"
Environment="NODE_TYPE=router"
WorkingDirectory=/opt/microdao/node-bootstrap
ExecStart=/usr/bin/python3 /opt/microdao/node-bootstrap/node_bootstrap.py
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
```
Запустити:
```bash
sudo systemctl daemon-reload
sudo systemctl enable node-bootstrap
sudo systemctl start node-bootstrap
sudo systemctl status node-bootstrap
```
### Як launchd service (macOS)
Створити файл `~/Library/LaunchAgents/com.daarion.node-bootstrap.plist`:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.daarion.node-bootstrap</string>
<key>ProgramArguments</key>
<array>
<string>/usr/bin/python3</string>
<string>/Users/apple/github-projects/microdao-daarion/services/node-registry/bootstrap/node_bootstrap.py</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>NODE_REGISTRY_URL</key>
<string>http://144.76.224.179:9205</string>
<key>NODE_ROLE</key>
<string>development</string>
<key>NODE_TYPE</key>
<string>router</string>
</dict>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardOutPath</key>
<string>/tmp/node-bootstrap.log</string>
<key>StandardErrorPath</key>
<string>/tmp/node-bootstrap.error.log</string>
</dict>
</plist>
```
Запустити:
```bash
launchctl load ~/Library/LaunchAgents/com.daarion.node-bootstrap.plist
launchctl start com.daarion.node-bootstrap
launchctl list | grep daarion
```
## Що робить агент?
1. **Збирає інформацію про систему**: CPU, RAM, диск, GPU, IP адреси
2. **Виявляє capabilities**: Docker, Ollama, GPU, доступні моделі
3. **Реєструє ноду** в Node Registry
4. **Підтримує heartbeat** кожні 30 секунд
5. **Автоматично перереєструється** якщо зв'язок втрачено
## Змінні середовища
- `NODE_REGISTRY_URL` - URL Node Registry (default: `http://localhost:9205`)
- `NODE_ROLE` - Роль ноди: `production`, `development`, `backup`, `worker` (default: `worker`)
- `NODE_TYPE` - Тип ноди: `router`, `gateway`, `worker`, `orchestrator` (default: `worker`)
- `HEARTBEAT_INTERVAL` - Інтервал heartbeat в секундах (default: `30`)
## Логи
Агент виводить детальні логи:
```
2025-11-23 10:00:00 - __main__ - INFO - 🚀 Initializing Node Bootstrap
2025-11-23 10:00:00 - __main__ - INFO - 📡 Registry URL: http://localhost:9205
2025-11-23 10:00:01 - __main__ - INFO - 📝 Registering node with registry...
2025-11-23 10:00:02 - __main__ - INFO - ✅ Node registered successfully: node-macbook-pro-a1b2c3d4
2025-11-23 10:00:02 - __main__ - INFO - 💓 Starting heartbeat loop (interval: 30s)
2025-11-23 10:00:32 - __main__ - DEBUG - 💓 Heartbeat sent: CPU=15.2% MEM=45.8%
```

288
services/node-registry/bootstrap/node_bootstrap.py Normal file
View File

@@ -0,0 +1,288 @@
#!/usr/bin/env python3
"""
Node Bootstrap Agent
Automatically registers the node and maintains heartbeat
"""
import os
import sys
import time
import socket
import platform
import psutil
import json
import logging
from typing import Dict, Any, Optional
from datetime import datetime
import requests
# Setup logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
class NodeBootstrap:
"""
Bootstrap agent that registers and maintains node presence in the registry
"""
def __init__(
self,
registry_url: str = "http://localhost:9205",
node_role: str = "worker",
node_type: str = "worker",
heartbeat_interval: int = 30,
auto_detect: bool = True
):
self.registry_url = registry_url.rstrip('/')
self.node_role = node_role
self.node_type = node_type
self.heartbeat_interval = heartbeat_interval
self.auto_detect = auto_detect
self.node_id = None
self.registered = False
logger.info(f"🚀 Initializing Node Bootstrap")
logger.info(f"📡 Registry URL: {self.registry_url}")
def get_system_info(self) -> Dict[str, Any]:
"""Collect system information"""
try:
hostname = socket.gethostname()
# Get local IP
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
local_ip = s.getsockname()[0]
s.close()
except:
local_ip = "127.0.0.1"
# Get public IP (if possible)
try:
public_ip = requests.get('https://api.ipify.org', timeout=5).text
except:
public_ip = None
# System specs
cpu_count = psutil.cpu_count()
memory = psutil.virtual_memory()
disk = psutil.disk_usage('/')
return {
"hostname": hostname,
"local_ip": local_ip,
"public_ip": public_ip,
"platform": platform.system(),
"platform_version": platform.version(),
"architecture": platform.machine(),
"cpu_count": cpu_count,
"memory_total_gb": round(memory.total / (1024**3), 2),
"disk_total_gb": round(disk.total / (1024**3), 2),
"python_version": platform.python_version(),
}
except Exception as e:
logger.error(f"Failed to collect system info: {e}")
return {}
def get_capabilities(self) -> Dict[str, Any]:
"""Detect node capabilities"""
capabilities = {
"system": self.get_system_info(),
"services": [],
"features": [],
}
# Check for Docker
try:
import subprocess
result = subprocess.run(['docker', '--version'], capture_output=True, timeout=5)
if result.returncode == 0:
capabilities["features"].append("docker")
except:
pass
# Check for GPU (NVIDIA)
try:
import subprocess
result = subprocess.run(['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
capture_output=True, timeout=5)
if result.returncode == 0:
gpu_names = result.stdout.decode().strip().split('\n')
capabilities["gpu"] = {
"available": True,
"gpus": gpu_names,
"count": len(gpu_names)
}
capabilities["features"].append("gpu")
except:
capabilities["gpu"] = {"available": False}
# Check for Ollama
try:
response = requests.get('http://localhost:11434/api/tags', timeout=5)
if response.status_code == 200:
models = response.json().get('models', [])
capabilities["ollama"] = {
"available": True,
"models": [m['name'] for m in models]
}
capabilities["services"].append("ollama")
except:
capabilities["ollama"] = {"available": False}
return capabilities
def register(self) -> bool:
"""Register node with registry"""
logger.info("📝 Registering node with registry...")
system_info = self.get_system_info()
capabilities = self.get_capabilities()
# Generate node name
hostname = system_info.get('hostname', 'unknown')
node_name = f"{hostname} ({self.node_role})"
payload = {
"node_name": node_name,
"node_role": self.node_role,
"node_type": self.node_type,
"hostname": hostname,
"ip_address": system_info.get('public_ip'),
"local_ip": system_info.get('local_ip'),
"capabilities": capabilities,
}
try:
response = requests.post(
f"{self.registry_url}/api/v1/nodes/register",
json=payload,
timeout=10
)
if response.status_code == 200:
data = response.json()
self.node_id = data.get('node_id')
self.registered = True
logger.info(f"✅ Node registered successfully: {self.node_id}")
logger.info(f"📊 Node details: {json.dumps(data, indent=2)}")
return True
else:
logger.error(f"❌ Registration failed: {response.status_code} - {response.text}")
return False
except Exception as e:
logger.error(f"❌ Registration error: {e}")
return False
def send_heartbeat(self) -> bool:
"""Send heartbeat to registry"""
if not self.registered or not self.node_id:
logger.warning("⚠️ Node not registered, skipping heartbeat")
return False
# Collect current metrics
cpu_percent = psutil.cpu_percent(interval=1)
memory = psutil.virtual_memory()
disk = psutil.disk_usage('/')
metrics = {
"cpu_percent": cpu_percent,
"memory_percent": memory.percent,
"memory_available_gb": round(memory.available / (1024**3), 2),
"disk_percent": disk.percent,
"disk_free_gb": round(disk.free / (1024**3), 2),
"timestamp": datetime.utcnow().isoformat() + "Z",
}
payload = {
"node_id": self.node_id,
"status": "online",
"metrics": metrics,
}
try:
response = requests.post(
f"{self.registry_url}/api/v1/nodes/heartbeat",
json=payload,
timeout=10
)
if response.status_code == 200:
logger.debug(f"💓 Heartbeat sent: CPU={cpu_percent}% MEM={memory.percent}%")
return True
else:
logger.warning(f"⚠️ Heartbeat failed: {response.status_code}")
return False
except Exception as e:
logger.error(f"❌ Heartbeat error: {e}")
return False
def run(self):
"""
Main loop: register and maintain heartbeat
"""
logger.info("🏁 Starting Node Bootstrap Agent")
# Initial registration
if not self.register():
logger.error("❌ Initial registration failed, exiting")
sys.exit(1)
# Heartbeat loop
logger.info(f"💓 Starting heartbeat loop (interval: {self.heartbeat_interval}s)")
consecutive_failures = 0
max_failures = 5
try:
while True:
time.sleep(self.heartbeat_interval)
if self.send_heartbeat():
consecutive_failures = 0
else:
consecutive_failures += 1
logger.warning(f"⚠️ Consecutive failures: {consecutive_failures}/{max_failures}")
if consecutive_failures >= max_failures:
logger.error("❌ Too many failures, attempting re-registration")
if self.register():
consecutive_failures = 0
else:
logger.error("❌ Re-registration failed, exiting")
sys.exit(1)
except KeyboardInterrupt:
logger.info("👋 Shutting down bootstrap agent")
sys.exit(0)
def main():
"""Main entry point"""
# Configuration from environment
registry_url = os.getenv("NODE_REGISTRY_URL", "http://localhost:9205")
node_role = os.getenv("NODE_ROLE", "worker")
node_type = os.getenv("NODE_TYPE", "worker")
heartbeat_interval = int(os.getenv("HEARTBEAT_INTERVAL", "30"))
# Create and run bootstrap agent
bootstrap = NodeBootstrap(
registry_url=registry_url,
node_role=node_role,
node_type=node_type,
heartbeat_interval=heartbeat_interval,
)
bootstrap.run()
if __name__ == "__main__":
main()

3
services/node-registry/bootstrap/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
requests>=2.31.0
psutil>=5.9.0

1
services/node-registry/requirements.txt
View File

@@ -8,3 +8,4 @@ sqlalchemy[asyncio]==2.0.36
alembic==1.14.0
python-json-logger==3.2.1
prometheus-client==0.21.0
psycopg2-binary>=2.9.0

80
services/node-registry/test_local.sh Executable file
View File

@@ -0,0 +1,80 @@
#!/bin/bash
# Test Node Registry locally with SQLite
set -e
echo "🧪 Testing Node Registry Service (SQLite)"
echo ""
# Set environment for SQLite
export NODE_REGISTRY_ENV="development"
export NODE_REGISTRY_HTTP_PORT="9205"
export NODE_REGISTRY_DB_FILE="test_node_registry.db"
# Clean up old database
rm -f test_node_registry.db
echo "✅ Cleaned up old database"
# Start Node Registry in background
echo "🚀 Starting Node Registry Service..."
cd "$(dirname "$0")"
# Use SQLite database module
cp app/database_sqlite.py app/database.py
python3 -m uvicorn app.main:app --host 0.0.0.0 --port 9205 --reload &
REGISTRY_PID=$!
echo "📝 Node Registry PID: $REGISTRY_PID"
# Wait for service to start
echo "⏳ Waiting for service to start..."
sleep 5
# Test health endpoint
echo ""
echo "1️⃣ Testing /health endpoint:"
curl -s http://localhost:9205/health | python3 -m json.tool
# Test bootstrap agent
echo ""
echo "2️⃣ Installing bootstrap dependencies..."
cd bootstrap
pip3 install -q -r requirements.txt
echo ""
echo "3️⃣ Starting bootstrap agent..."
export NODE_REGISTRY_URL="http://localhost:9205"
export NODE_ROLE="development"
export NODE_TYPE="router"
export HEARTBEAT_INTERVAL="10"
# Run bootstrap for 30 seconds
timeout 30 python3 node_bootstrap.py &
BOOTSTRAP_PID=$!
echo "📝 Bootstrap PID: $BOOTSTRAP_PID"
# Wait for registration
sleep 15
# Check nodes
echo ""
echo "4️⃣ Checking registered nodes:"
curl -s http://localhost:9205/api/v1/nodes | python3 -m json.tool
# Check network stats
echo ""
echo "5️⃣ Network statistics:"
curl -s http://localhost:9205/metrics | python3 -m json.tool
# Cleanup
echo ""
echo "🧹 Cleaning up..."
kill $REGISTRY_PID 2>/dev/null || true
kill $BOOTSTRAP_PID 2>/dev/null || true
echo ""
echo "✅ Test completed!"
echo ""
echo "📊 Database file: test_node_registry.db"
echo "💡 To inspect: sqlite3 test_node_registry.db"