ed7ad49d3a
P3.2 — Multi-node deployment: - Added node-worker service to docker-compose.node1.yml (NODE_ID=noda1) - NCS NODA1 now has NODE_WORKER_URL for metrics collection - Fixed NODE_ID consistency: router NODA1 uses 'noda1' - NODA2 node-worker/NCS gets NCS_REPORT_URL for latency reporting P3.3 — NATS accounts/auth (opt-in config): - config/nats-server.conf with 3 accounts: SYS, FABRIC, APP - Per-user topic permissions (router, ncs, node_worker) - Leafnode listener :7422 with auth - Not yet activated (requires credential provisioning) P3.4 — Prometheus counters: - Router /fabric_metrics: caps_refresh, caps_stale, model_select, offload_total, breaker_state, score_ms histogram - Node Worker /prom_metrics: jobs_total, inflight gauge, latency_ms histogram - NCS /prom_metrics: runtime_health, runtime_p50/p95, node_wait_ms - All bound to 127.0.0.1 (not externally exposed) Made-with: Cursor
69 lines
1.7 KiB
Python
69 lines
1.7 KiB
Python
"""Node Worker — NATS offload executor for cross-node inference."""
|
|
import logging
|
|
import os
|
|
|
|
from fastapi import FastAPI
|
|
|
|
import config
|
|
import worker
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("node-worker")
|
|
|
|
app = FastAPI(title="Node Worker", version="1.0.0")
|
|
|
|
_nats_client = None
|
|
|
|
|
|
@app.get("/healthz")
|
|
async def healthz():
|
|
connected = _nats_client is not None and _nats_client.is_connected if _nats_client else False
|
|
return {
|
|
"status": "ok" if connected else "degraded",
|
|
"node_id": config.NODE_ID,
|
|
"nats_connected": connected,
|
|
"max_concurrency": config.MAX_CONCURRENCY,
|
|
}
|
|
|
|
|
|
@app.get("/metrics")
|
|
async def metrics():
|
|
return worker.get_metrics()
|
|
|
|
|
|
@app.get("/prom_metrics")
|
|
async def prom_metrics():
|
|
from fastapi.responses import Response
|
|
import fabric_metrics as fm
|
|
data = fm.get_metrics_text()
|
|
if data:
|
|
return Response(content=data, media_type="text/plain; charset=utf-8")
|
|
return {"error": "prometheus_client not installed"}
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup():
|
|
global _nats_client
|
|
try:
|
|
import nats as nats_lib
|
|
_nats_client = await nats_lib.connect(config.NATS_URL)
|
|
logger.info(f"✅ NATS connected: {config.NATS_URL}")
|
|
await worker.start(_nats_client)
|
|
logger.info(f"✅ Node Worker ready: node={config.NODE_ID} concurrency={config.MAX_CONCURRENCY}")
|
|
except Exception as e:
|
|
logger.error(f"❌ Startup failed: {e}")
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown():
|
|
if _nats_client:
|
|
try:
|
|
await _nats_client.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=config.PORT)
|