P3.1: GPU/Queue-aware routing — NCS metrics + scoring-based model selection

NCS (services/node-capabilities/metrics.py): - NodeLoad: inflight_jobs, queue_depth, concurrency_limit, estimated_wait_ms, cpu_load_1m, mem_pressure (macOS + Linux), rtt_ms_to_hub - RuntimeLoad: per-runtime healthy, p50_ms, p95_ms from rolling 50-sample window - POST /capabilities/report_latency for node-worker → NCS reporting - NCS fetches worker metrics via NODE_WORKER_URL Node Worker: - GET /metrics endpoint (inflight, concurrency, latency buffers) - Latency tracking per job type (llm/vision) with rolling buffer - Fire-and-forget latency reporting to NCS after each successful job Router (model_select v3): - score_candidate(): wait + model_latency + cross_node_penalty + prefer_bonus - LOCAL_THRESHOLD_MS=250: prefer local if within threshold of remote - ModelSelection.score field for observability - Structured [score] logs with chosen node, model, and score breakdown Tests: 19 new (12 scoring + 7 NCS metrics), 36 total pass Docs: ops/runbook_p3_1.md, ops/CHANGELOG_FABRIC.md No breaking changes to JobRequest/JobResponse or capabilities schema. Made-with: Cursor
2026-02-27 02:55:44 -08:00
parent c4b94a327d
commit a605b8c43e
11 changed files with 706 additions and 40 deletions
--- a/tests/test_ncs_metrics.py
+++ b/tests/test_ncs_metrics.py
@@ -0,0 +1,69 @@
+"""Tests for NCS metrics module."""
+import sys
+import os
+import asyncio
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "services", "node-capabilities"))
+
+from metrics import (
+    record_latency, get_latency_stats, get_cpu_load, get_mem_pressure,
+    build_node_load, build_runtime_load, _latency_buffer,
+)
+
+
+def setup_function():
+    _latency_buffer.clear()
+
+
+def test_record_and_get_latency():
+    record_latency("ollama", "llm", 500)
+    record_latency("ollama", "llm", 300)
+    record_latency("ollama", "llm", 700)
+    stats = get_latency_stats("ollama", "llm")
+    assert stats["samples"] == 3
+    assert stats["p50_ms"] == 500
+    assert stats["p95_ms"] == 700
+
+
+def test_empty_latency_stats():
+    stats = get_latency_stats("nonexistent", "llm")
+    assert stats["p50_ms"] is None
+    assert stats["samples"] == 0
+
+
+def test_cpu_load_returns_float_or_none():
+    result = get_cpu_load()
+    assert result is None or isinstance(result, float)
+
+
+def test_mem_pressure_returns_valid_or_none():
+    result = get_mem_pressure()
+    assert result is None or result in ("low", "medium", "high", "critical")
+
+
+def test_build_node_load_defaults():
+    result = asyncio.run(build_node_load(worker_metrics={
+        "inflight_jobs": 0, "concurrency_limit": 2, "queue_depth": 0,
+    }))
+    assert result["inflight_jobs"] == 0
+    assert result["estimated_wait_ms"] == 0
+    assert result["concurrency_limit"] == 2
+    assert "ts" in result
+
+
+def test_build_node_load_wait_when_busy():
+    record_latency("ollama", "llm", 1000)
+    result = asyncio.run(build_node_load(worker_metrics={
+        "inflight_jobs": 5, "concurrency_limit": 2, "queue_depth": 0,
+    }))
+    assert result["estimated_wait_ms"] == 4 * 1000
+
+
+def test_build_runtime_load():
+    runtimes = {"ollama": {"status": "ok"}, "swapper": {"status": "error: timeout"}}
+    result = asyncio.run(build_runtime_load(runtimes))
+    assert len(result) == 2
+    ollama_rl = next(r for r in result if r["runtime"] == "ollama")
+    assert ollama_rl["healthy"] is True
+    swapper_rl = next(r for r in result if r["runtime"] == "swapper")
+    assert swapper_rl["healthy"] is False
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -0,0 +1,177 @@
+"""Tests for P3.1 scoring-based model selection."""
+import sys
+import os
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "services", "router"))
+
+from model_select import (
+    score_candidate,
+    select_best_model,
+    ProfileRequirements,
+    ModelSelection,
+    LOCAL_THRESHOLD_MS,
+)
+
+
+def _caps(served, node_load=None, runtime_load=None, nodes=None):
+    return {
+        "served_models": served,
+        "node_load": node_load or {},
+        "runtime_load": runtime_load or [],
+        "nodes": nodes or {},
+    }
+
+
+def _model(name, typ="llm", local=True, node="n1", runtime="ollama", **kw):
+    return {"name": name, "type": typ, "local": local, "node": node,
+            "runtime": runtime, "base_url": "http://x", **kw}
+
+
+def _reqs(typ="llm", prefer=None):
+    return ProfileRequirements("test", typ, prefer or [])
+
+
+# ── 1) local wins when scores close ────────────────────────────────────────
+
+def test_local_wins_when_scores_close():
+    caps = _caps(
+        served=[
+            _model("qwen3:14b", local=True, node="n1"),
+            _model("qwen3:14b", local=False, node="n2"),
+        ],
+        node_load={"estimated_wait_ms": 0, "rtt_ms_to_hub": None},
+    )
+    sel = select_best_model(_reqs(), caps)
+    assert sel is not None
+    assert sel.local is True
+    assert sel.node == "n1"
+
+
+# ── 2) remote wins when local wait is high ─────────────────────────────────
+
+def test_remote_wins_when_local_wait_high():
+    caps = _caps(
+        served=[
+            _model("qwen3:14b", local=True, node="n1"),
+            _model("qwen3:14b", local=False, node="n2"),
+        ],
+        node_load={"estimated_wait_ms": 5000, "rtt_ms_to_hub": None},
+        nodes={"n2": {"node_id": "n2", "node_load": {"estimated_wait_ms": 0, "rtt_ms_to_hub": 50}}},
+    )
+    sel = select_best_model(_reqs(), caps)
+    assert sel is not None
+    assert sel.local is False
+    assert sel.node == "n2"
+
+
+# ── 3) exclude_nodes works ─────────────────────────────────────────────────
+
+def test_exclude_nodes_works():
+    caps = _caps(served=[
+        _model("qwen3:14b", local=False, node="n2"),
+        _model("qwen3:14b", local=False, node="n3"),
+    ])
+    sel = select_best_model(_reqs(), caps, exclude_nodes={"n2"})
+    assert sel is not None
+    assert sel.node == "n3"
+
+
+# ── 4) breaker open → node excluded (via exclude_nodes) ───────────────────
+
+def test_breaker_excludes_node():
+    caps = _caps(served=[
+        _model("qwen3:14b", local=False, node="broken"),
+        _model("qwen3:14b", local=True, node="n1"),
+    ])
+    sel = select_best_model(_reqs(), caps, exclude_nodes={"broken"})
+    assert sel is not None
+    assert sel.node == "n1"
+
+
+# ── 5) required_type filter ────────────────────────────────────────────────
+
+def test_required_type_filter():
+    caps = _caps(served=[
+        _model("qwen3:14b", typ="llm"),
+        _model("llava:13b", typ="vision"),
+    ])
+    sel = select_best_model(_reqs(typ="vision"), caps)
+    assert sel is not None
+    assert sel.name == "llava:13b"
+
+
+# ── 6) prefer list filter ─────────────────────────────────────────────────
+
+def test_prefer_list_selects_preferred():
+    caps = _caps(served=[
+        _model("qwen3:14b"),
+        _model("qwen3.5:35b"),
+    ])
+    sel = select_best_model(_reqs(prefer=["qwen3.5:35b"]), caps)
+    assert sel is not None
+    assert sel.name == "qwen3.5:35b"
+
+
+# ── 7) score formula — prefer bonus lowers score ──────────────────────────
+
+def test_prefer_bonus_lowers_score():
+    m1 = _model("qwen3:14b")
+    m2 = _model("qwen3.5:35b")
+    caps = _caps(served=[m1, m2])
+    s1 = score_candidate(m1, caps, prefer=["qwen3:14b"])
+    s2 = score_candidate(m2, caps, prefer=["qwen3:14b"])
+    assert s1 < s2
+
+
+# ── 8) score formula — cross_penalty for remote ──────────────────────────
+
+def test_cross_penalty_for_remote():
+    local = _model("m", local=True)
+    remote = _model("m", local=False, node="r1")
+    caps = _caps(served=[local, remote])
+    sl = score_candidate(local, caps, prefer=[])
+    sr = score_candidate(remote, caps, prefer=[], rtt_hint_ms=50)
+    assert sr > sl
+
+
+# ── 9) score formula — wait increases score ──────────────────────────────
+
+def test_wait_increases_score():
+    m = _model("m", local=True)
+    caps_idle = _caps(served=[m], node_load={"estimated_wait_ms": 0})
+    caps_busy = _caps(served=[m], node_load={"estimated_wait_ms": 3000})
+    s_idle = score_candidate(m, caps_idle, prefer=[])
+    s_busy = score_candidate(m, caps_busy, prefer=[])
+    assert s_busy > s_idle
+
+
+# ── 10) no candidates → None ─────────────────────────────────────────────
+
+def test_no_candidates_returns_none():
+    caps = _caps(served=[_model("m", typ="stt")])
+    sel = select_best_model(_reqs(typ="llm"), caps)
+    assert sel is None
+
+
+# ── 11) local threshold: local wins within threshold even if remote lower ─
+
+def test_local_threshold():
+    caps = _caps(
+        served=[
+            _model("qwen3:14b", local=True, node="n1"),
+            _model("qwen3:14b", local=False, node="n2"),
+        ],
+        node_load={"estimated_wait_ms": 100},
+        nodes={"n2": {"node_id": "n2", "node_load": {"estimated_wait_ms": 0, "rtt_ms_to_hub": 10}}},
+    )
+    sel = select_best_model(_reqs(), caps)
+    assert sel.local is True
+
+
+# ── 12) code type cross-filters with llm ─────────────────────────────────
+
+def test_code_type_finds_llm_models():
+    caps = _caps(served=[_model("qwen3:14b", typ="llm")])
+    sel = select_best_model(_reqs(typ="code"), caps)
+    assert sel is not None
+    assert sel.name == "qwen3:14b"