P2.2+P2.3: NATS offload node-worker + router offload integration

Node Worker (services/node-worker/):
- NATS subscriber for node.{NODE_ID}.llm.request / vision.request
- Canonical JobRequest/JobResponse envelope (Pydantic)
- Idempotency cache (TTL 10min) with inflight dedup
- Deadline enforcement (DEADLINE_EXCEEDED on expired jobs)
- Concurrency limiter (semaphore, returns busy)
- Ollama + Swapper vision providers

Router offload (services/router/offload_client.py):
- NATS req/reply with configurable retries
- Circuit breaker per node+type (3 fails/60s → open 120s)
- Concurrency semaphore for remote requests

Model selection (services/router/model_select.py):
- exclude_nodes parameter for circuit-broken nodes
- force_local flag for fallback re-selection
- Integrated circuit breaker state awareness

Router /infer pipeline:
- Remote offload path when NCS selects remote node
- Automatic fallback: exclude failed node → force_local re-select
- Deadline propagation from router to node-worker

Tests: 17 unit tests (idempotency, deadline, circuit breaker)
Docs: ops/offload_routing.md (subjects, envelope, verification)
Made-with: Cursor

services/router/offload_client.py

@@ -0,0 +1,153 @@
+"""NATS offload client — sends inference requests to remote nodes with
+circuit breaker, retries, and deadline enforcement."""
+import asyncio
+import json
+import logging
+import os
+import time
+from typing import Any, Dict, Literal, Optional, Set
+
+logger = logging.getLogger("offload_client")
+
+CB_FAILS = int(os.getenv("ROUTER_OFFLOAD_CB_FAILS", "3"))
+CB_WINDOW_S = int(os.getenv("ROUTER_OFFLOAD_CB_WINDOW_S", "60"))
+CB_OPEN_S = int(os.getenv("ROUTER_OFFLOAD_CB_OPEN_S", "120"))
+MAX_RETRIES = int(os.getenv("ROUTER_OFFLOAD_RETRIES", "1"))
+MAX_CONCURRENCY = int(os.getenv("ROUTER_OFFLOAD_MAX_CONCURRENCY_REMOTE", "8"))
+
+_semaphore: Optional[asyncio.Semaphore] = None
+_breakers: Dict[str, Dict[str, Any]] = {}
+
+
+def _get_semaphore() -> asyncio.Semaphore:
+    global _semaphore
+    if _semaphore is None:
+        _semaphore = asyncio.Semaphore(MAX_CONCURRENCY)
+    return _semaphore
+
+
+def _breaker_key(node_id: str, req_type: str) -> str:
+    return f"{node_id}:{req_type}"
+
+
+def is_node_available(node_id: str, req_type: str) -> bool:
+    key = _breaker_key(node_id, req_type)
+    state = _breakers.get(key)
+    if not state:
+        return True
+    open_until = state.get("open_until", 0)
+    if open_until > time.time():
+        return False
+    if open_until > 0 and open_until <= time.time():
+        return True
+    now = time.time()
+    window_start = now - CB_WINDOW_S
+    recent = [t for t in state.get("failures", []) if t > window_start]
+    state["failures"] = recent
+    return len(recent) < CB_FAILS
+
+
+def record_failure(node_id: str, req_type: str):
+    key = _breaker_key(node_id, req_type)
+    state = _breakers.setdefault(key, {"failures": [], "open_until": 0})
+    state["failures"].append(time.time())
+    window_start = time.time() - CB_WINDOW_S
+    recent = [t for t in state["failures"] if t > window_start]
+    state["failures"] = recent
+    if len(recent) >= CB_FAILS:
+        state["open_until"] = time.time() + CB_OPEN_S
+        logger.warning(f"Circuit OPEN: {key} ({len(recent)} failures in {CB_WINDOW_S}s, open for {CB_OPEN_S}s)")
+
+
+def record_success(node_id: str, req_type: str):
+    key = _breaker_key(node_id, req_type)
+    state = _breakers.get(key)
+    if state:
+        state["failures"] = []
+        state["open_until"] = 0
+
+
+def get_unavailable_nodes(req_type: str) -> Set[str]:
+    result = set()
+    for key, state in _breakers.items():
+        if not key.endswith(f":{req_type}"):
+            continue
+        nid = key.rsplit(":", 1)[0]
+        if not is_node_available(nid, req_type):
+            result.add(nid)
+    return result
+
+
+async def offload_infer(
+    nats_client,
+    node_id: str,
+    required_type: Literal["llm", "vision", "stt", "tts"],
+    job_payload: Dict[str, Any],
+    timeout_ms: int = 25000,
+) -> Optional[Dict[str, Any]]:
+    """Send inference job to remote node via NATS request/reply.
+
+    Returns parsed JobResponse dict or None on total failure.
+    Retries on transient errors (timeout, busy). Does NOT retry on provider errors.
+    """
+    subject = f"node.{node_id.lower()}.{required_type}.request"
+    payload_bytes = json.dumps(job_payload).encode()
+    sem = _get_semaphore()
+
+    for attempt in range(1 + MAX_RETRIES):
+        timeout_s = timeout_ms / 1000.0
+        if timeout_s <= 0:
+            logger.warning(f"[offload] deadline exhausted before attempt {attempt}")
+            return None
+
+        t0 = time.time()
+        try:
+            async with sem:
+                logger.info(
+                    f"[offload.start] node={node_id} subj={subject} "
+                    f"timeout={timeout_ms}ms attempt={attempt}"
+                )
+                msg = await nats_client.request(subject, payload_bytes, timeout=timeout_s)
+                resp = json.loads(msg.data)
+
+            latency = int((time.time() - t0) * 1000)
+            status = resp.get("status", "error")
+
+            if status == "ok":
+                record_success(node_id, required_type)
+                logger.info(
+                    f"[offload.done] node={node_id} status=ok latency={latency}ms "
+                    f"provider={resp.get('provider')} model={resp.get('model')} "
+                    f"cached={resp.get('cached', False)}"
+                )
+                return resp
+
+            if status in ("timeout", "busy") and attempt < MAX_RETRIES:
+                elapsed = int((time.time() - t0) * 1000)
+                timeout_ms -= elapsed
+                logger.warning(f"[offload.retry] node={node_id} status={status} → retry {attempt+1}")
+                continue
+
+            record_failure(node_id, required_type)
+            logger.warning(
+                f"[offload.fail] node={node_id} status={status} "
+                f"error={resp.get('error', {}).get('code', '?')}"
+            )
+            return resp
+
+        except asyncio.TimeoutError:
+            record_failure(node_id, required_type)
+            elapsed = int((time.time() - t0) * 1000)
+            timeout_ms -= elapsed
+            if attempt < MAX_RETRIES:
+                logger.warning(f"[offload.timeout] node={node_id} {elapsed}ms → retry {attempt+1}")
+                continue
+            logger.warning(f"[offload.timeout] node={node_id} all retries exhausted")
+            return None
+
+        except Exception as e:
+            record_failure(node_id, required_type)
+            logger.warning(f"[offload.error] node={node_id} {e}")
+            return None
+
+    return None