c4b94a327d
Node Worker (services/node-worker/):
- NATS subscriber for node.{NODE_ID}.llm.request / vision.request
- Canonical JobRequest/JobResponse envelope (Pydantic)
- Idempotency cache (TTL 10min) with inflight dedup
- Deadline enforcement (DEADLINE_EXCEEDED on expired jobs)
- Concurrency limiter (semaphore, returns busy)
- Ollama + Swapper vision providers
Router offload (services/router/offload_client.py):
- NATS req/reply with configurable retries
- Circuit breaker per node+type (3 fails/60s → open 120s)
- Concurrency semaphore for remote requests
Model selection (services/router/model_select.py):
- exclude_nodes parameter for circuit-broken nodes
- force_local flag for fallback re-selection
- Integrated circuit breaker state awareness
Router /infer pipeline:
- Remote offload path when NCS selects remote node
- Automatic fallback: exclude failed node → force_local re-select
- Deadline propagation from router to node-worker
Tests: 17 unit tests (idempotency, deadline, circuit breaker)
Docs: ops/offload_routing.md (subjects, envelope, verification)
Made-with: Cursor
53 lines
1.7 KiB
Python
53 lines
1.7 KiB
Python
"""Tests for offload_client circuit breaker."""
|
|
import sys
|
|
import os
|
|
import time
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "services", "router"))
|
|
|
|
import offload_client
|
|
|
|
|
|
def setup_function():
|
|
offload_client._breakers.clear()
|
|
|
|
|
|
def test_node_available_by_default():
|
|
assert offload_client.is_node_available("noda99", "llm") is True
|
|
|
|
|
|
def test_node_unavailable_after_threshold():
|
|
for _ in range(offload_client.CB_FAILS):
|
|
offload_client.record_failure("noda99", "llm")
|
|
assert offload_client.is_node_available("noda99", "llm") is False
|
|
|
|
|
|
def test_node_available_after_open_expires():
|
|
for _ in range(offload_client.CB_FAILS):
|
|
offload_client.record_failure("noda99", "llm")
|
|
offload_client._breakers["noda99:llm"]["open_until"] = time.time() - 1
|
|
assert offload_client.is_node_available("noda99", "llm") is True
|
|
|
|
|
|
def test_success_resets_breaker():
|
|
for _ in range(offload_client.CB_FAILS):
|
|
offload_client.record_failure("noda99", "llm")
|
|
offload_client._breakers["noda99:llm"]["open_until"] = time.time() - 1
|
|
offload_client.record_success("noda99", "llm")
|
|
assert offload_client.is_node_available("noda99", "llm") is True
|
|
|
|
|
|
def test_get_unavailable_nodes():
|
|
for _ in range(offload_client.CB_FAILS):
|
|
offload_client.record_failure("bad_node", "llm")
|
|
unavail = offload_client.get_unavailable_nodes("llm")
|
|
assert "bad_node" in unavail
|
|
assert offload_client.get_unavailable_nodes("vision") == set()
|
|
|
|
|
|
def test_different_types_independent():
|
|
for _ in range(offload_client.CB_FAILS):
|
|
offload_client.record_failure("noda99", "llm")
|
|
assert offload_client.is_node_available("noda99", "llm") is False
|
|
assert offload_client.is_node_available("noda99", "vision") is True
|