""" Voice HA (PR1 + PR2 + PR3) contract tests. Tests enforce: 1. Node Worker voice capability flags (voice_tts, voice_llm, voice_stt) 2. Voice semaphore isolation from generic concurrency 3. Router voice scoring: prefer local, penalise high load 4. offload_client supports voice.* subject patterns 5. BFF VOICE_HA_ENABLED feature flag routes correctly """ import asyncio import importlib import importlib.util import os import sys import types import unittest from unittest.mock import AsyncMock, MagicMock, patch # ── helpers ────────────────────────────────────────────────────────────────── def _reload_config(overrides: dict): """Reload node-worker config with env overrides.""" worker_path = os.path.join(os.path.dirname(__file__), "..", "services", "node-worker") if worker_path not in sys.path: sys.path.insert(0, worker_path) for k, v in overrides.items(): os.environ[k] = v try: import config as _cfg importlib.reload(_cfg) return _cfg finally: pass # caller restores env if needed # ═══════════════════════════════════════════════════════════════════════════ # PR1: Node Worker capability flags # ═══════════════════════════════════════════════════════════════════════════ class TestNodeWorkerVoiceCaps(unittest.TestCase): def setUp(self): self._worker_path = os.path.join( os.path.dirname(__file__), "..", "services", "node-worker" ) if self._worker_path not in sys.path: sys.path.insert(0, self._worker_path) def test_config_has_voice_concurrency_vars(self): """Voice HA concurrency env vars must exist in config.""" cfg = _reload_config({ "STT_PROVIDER": "memory_service", "TTS_PROVIDER": "memory_service", "VOICE_MAX_CONCURRENT_TTS": "4", "VOICE_MAX_CONCURRENT_LLM": "2", "VOICE_MAX_CONCURRENT_STT": "2", }) self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 4) self.assertEqual(cfg.VOICE_MAX_CONCURRENT_LLM, 2) self.assertEqual(cfg.VOICE_MAX_CONCURRENT_STT, 2) def test_config_has_voice_deadline_vars(self): cfg = _reload_config({ "VOICE_TTS_DEADLINE_MS": "3000", "VOICE_LLM_FAST_MS": "9000", "VOICE_LLM_QUALITY_MS": "12000", "VOICE_STT_DEADLINE_MS": "6000", }) self.assertEqual(cfg.VOICE_TTS_DEADLINE_MS, 3000) self.assertEqual(cfg.VOICE_LLM_FAST_MS, 9000) self.assertEqual(cfg.VOICE_STT_DEADLINE_MS, 6000) def test_voice_tts_false_when_tts_provider_none(self): """voice_tts SEMANTIC capability must be False when TTS_PROVIDER=none.""" cfg = _reload_config({"TTS_PROVIDER": "none"}) self.assertEqual(cfg.TTS_PROVIDER, "none") # Semantic: voice_tts = (TTS_PROVIDER != "none") voice_tts_cap = cfg.TTS_PROVIDER != "none" self.assertFalse(voice_tts_cap) def test_voice_tts_true_when_memory_service(self): """voice_tts SEMANTIC capability must be True when TTS_PROVIDER=memory_service.""" cfg = _reload_config({"TTS_PROVIDER": "memory_service"}) self.assertEqual(cfg.TTS_PROVIDER, "memory_service") # Semantic: provider configured → capability available (not NATS-dependent) voice_tts_cap = cfg.TTS_PROVIDER != "none" self.assertTrue(voice_tts_cap) def test_voice_cap_semantic_not_nats_dependent(self): """voice_* capability must NOT depend on NATS subscription state. Semantics = provider configured. Operational state (NATS active) is separate (runtime.nats_subscriptions). """ cfg = _reload_config({"TTS_PROVIDER": "memory_service", "STT_PROVIDER": "memory_service"}) # Even if NATS subscriptions were empty (reconnect / restart race), # semantic caps must still be True voice_tts_semantic = cfg.TTS_PROVIDER != "none" voice_stt_semantic = cfg.STT_PROVIDER != "none" self.assertTrue(voice_tts_semantic, "voice_tts semantic must be True regardless of NATS") self.assertTrue(voice_stt_semantic, "voice_stt semantic must be True regardless of NATS") def test_voice_concurrency_env_override(self): cfg = _reload_config({"VOICE_MAX_CONCURRENT_TTS": "8"}) self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 8) def test_voice_semaphores_independent_from_generic(self): """Voice semaphore limit must not equal MAX_CONCURRENCY (they are independent).""" cfg = _reload_config({ "NODE_WORKER_MAX_CONCURRENCY": "2", "VOICE_MAX_CONCURRENT_TTS": "4", }) self.assertEqual(cfg.MAX_CONCURRENCY, 2) self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 4) self.assertNotEqual(cfg.MAX_CONCURRENCY, cfg.VOICE_MAX_CONCURRENT_TTS) class TestNodeWorkerVoiceSubjects(unittest.TestCase): """Worker must register voice.* NATS subjects when providers configured.""" def setUp(self): self._worker_path = os.path.join( os.path.dirname(__file__), "..", "services", "node-worker" ) if self._worker_path not in sys.path: sys.path.insert(0, self._worker_path) def test_voice_subjects_set_exists(self): import worker self.assertTrue(hasattr(worker, "_VOICE_SUBJECTS")) self.assertIsInstance(worker._VOICE_SUBJECTS, set) def test_voice_semaphores_defined(self): import worker self.assertTrue(hasattr(worker, "_voice_sem_tts")) self.assertTrue(hasattr(worker, "_voice_sem_llm")) self.assertTrue(hasattr(worker, "_voice_sem_stt")) self.assertIsInstance(worker._voice_sem_tts, asyncio.Semaphore) def test_voice_semaphore_map_keys(self): import worker sem_map = worker._VOICE_SEMAPHORES self.assertIn("voice.tts", sem_map) self.assertIn("voice.llm", sem_map) self.assertIn("voice.stt", sem_map) class TestFabricMetricsVoice(unittest.TestCase): """fabric_metrics must expose voice HA metric helpers.""" def setUp(self): self._worker_path = os.path.join( os.path.dirname(__file__), "..", "services", "node-worker" ) if self._worker_path not in sys.path: sys.path.insert(0, self._worker_path) def test_voice_metric_functions_exist(self): import fabric_metrics as fm self.assertTrue(callable(fm.inc_voice_job)) self.assertTrue(callable(fm.set_voice_inflight)) self.assertTrue(callable(fm.observe_voice_latency)) def test_voice_metric_functions_no_raise(self): import fabric_metrics as fm # Should not raise regardless of prometheus_client availability fm.inc_voice_job("voice.tts", "ok") fm.set_voice_inflight("voice.tts", 1) fm.observe_voice_latency("voice.tts", 1500) # ═══════════════════════════════════════════════════════════════════════════ # PR2: Router offload_client voice subjects # ═══════════════════════════════════════════════════════════════════════════ class TestOffloadClientVoiceSubjects(unittest.TestCase): def setUp(self): self._router_path = os.path.join( os.path.dirname(__file__), "..", "services", "router" ) if self._router_path not in sys.path: sys.path.insert(0, self._router_path) def test_offload_voice_tts_uses_dotted_subject(self): """voice.tts → node.{id}.voice.tts.request (not node.{id}.tts.request).""" import offload_client # The subject is built as f"node.{node_id}.{required_type}.request" # So required_type="voice.tts" → "node.noda2.voice.tts.request" node_id = "noda2" req_type = "voice.tts" expected_subject = f"node.{node_id}.{req_type}.request" self.assertEqual(expected_subject, "node.noda2.voice.tts.request") def test_offload_generic_tts_uses_flat_subject(self): """Generic tts → node.{id}.tts.request (unchanged).""" req_type = "tts" node_id = "noda2" expected_subject = f"node.{node_id}.{req_type}.request" self.assertEqual(expected_subject, "node.noda2.tts.request") # Must differ from voice subject self.assertNotEqual("node.noda2.tts.request", "node.noda2.voice.tts.request") def test_offload_client_accepts_voice_type(self): """offload_infer signature must accept voice.* types (no Literal restriction).""" import inspect import offload_client sig = inspect.signature(offload_client.offload_infer) param = sig.parameters.get("required_type") self.assertIsNotNone(param, "required_type parameter must exist") # Must not be restricted to old Literal — annotation should be str or Any ann = param.annotation ann_str = str(ann) self.assertNotIn("Literal", ann_str, "required_type must not be Literal (voice.* types needed)") class TestRouterFabricMetricsVoice(unittest.TestCase): """Test router/fabric_metrics.py voice helper functions. Loads the module directly via importlib to avoid collision with node-worker's fabric_metrics that is earlier in sys.path. """ def _load_router_fm(self): router_path = os.path.join( os.path.dirname(__file__), "..", "services", "router", "fabric_metrics.py" ) spec = importlib.util.spec_from_file_location("router_fabric_metrics", router_path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def test_router_voice_metric_functions_exist(self): fm = self._load_router_fm() self.assertTrue(callable(fm.inc_voice_cap_request)) self.assertTrue(callable(fm.inc_voice_offload)) self.assertTrue(callable(fm.set_voice_breaker)) self.assertTrue(callable(fm.observe_voice_score)) def test_router_voice_metrics_no_raise(self): fm = self._load_router_fm() fm.inc_voice_cap_request("voice_tts", "ok") fm.inc_voice_offload("voice_tts", "noda2", "ok") fm.set_voice_breaker("voice_tts", "noda2", False) fm.observe_voice_score("voice_tts", 250.0) # ═══════════════════════════════════════════════════════════════════════════ # PR3: BFF config feature flag # ═══════════════════════════════════════════════════════════════════════════ class TestBFFVoiceHAConfig(unittest.TestCase): def setUp(self): self._console_path = os.path.join( os.path.dirname(__file__), "..", "services", "sofiia-console", "app" ) # Add parent for package import parent = os.path.join(os.path.dirname(__file__), "..", "services", "sofiia-console") if parent not in sys.path: sys.path.insert(0, parent) def _reload_bff_config(self, voice_ha: str = "false"): os.environ["VOICE_HA_ENABLED"] = voice_ha from app import config as bff_config importlib.reload(bff_config) return bff_config def test_voice_ha_disabled_by_default(self): os.environ.pop("VOICE_HA_ENABLED", None) from app import config as bff_config importlib.reload(bff_config) self.assertFalse(bff_config.is_voice_ha_enabled()) def test_voice_ha_enabled_via_env(self): cfg = self._reload_bff_config("true") self.assertTrue(cfg.is_voice_ha_enabled()) def test_voice_ha_disabled_explicit(self): cfg = self._reload_bff_config("false") self.assertFalse(cfg.is_voice_ha_enabled()) def test_voice_ha_enabled_variants(self): for val in ("1", "yes", "true", "True", "YES"): os.environ["VOICE_HA_ENABLED"] = val from app import config as bff_config importlib.reload(bff_config) self.assertTrue(bff_config.is_voice_ha_enabled(), f"Failed for value: {val}") def test_voice_ha_router_url_default(self): os.environ.pop("VOICE_HA_ROUTER_URL", None) os.environ.pop("NODES_NODA2_ROUTER_URL", None) from app import config as bff_config importlib.reload(bff_config) url = bff_config.get_voice_ha_router_url() # Must return a string URL, not empty self.assertIsInstance(url, str) self.assertTrue(url.startswith("http")) def test_voice_ha_router_url_override(self): os.environ["VOICE_HA_ROUTER_URL"] = "http://router-ha:9200" from app import config as bff_config importlib.reload(bff_config) url = bff_config.get_voice_ha_router_url() self.assertEqual(url, "http://router-ha:9200") del os.environ["VOICE_HA_ROUTER_URL"] def tearDown(self): os.environ.pop("VOICE_HA_ENABLED", None) os.environ.pop("VOICE_HA_ROUTER_URL", None) # ═══════════════════════════════════════════════════════════════════════════ # Voice scoring logic: prefer local, penalise high load # ═══════════════════════════════════════════════════════════════════════════ class TestVoiceHAScoringLogic(unittest.TestCase): """Unit tests for voice node scoring formula (inline, no Router import needed).""" def _score(self, node_load: dict, runtime_load: dict, router_node: str, node_id: str, prefer_bonus: int = 200) -> int: """Replicate the scoring formula from Router voice_capability_offload.""" nl = node_load rl = runtime_load wait_ms = nl.get("wait_ms", 0) or nl.get("inflight", 0) * 50 rtt_ms = nl.get("rtt_ms", 0) p95_ms = rl.get("p95_ms", 0) if rl else 0 mem_penalty = 300 if nl.get("mem_pressure") == "high" else 0 local_bonus = prefer_bonus if node_id.lower() == router_node.lower() else 0 return wait_ms + rtt_ms + p95_ms + mem_penalty - local_bonus def test_local_node_preferred_when_load_similar(self): local_score = self._score( {"wait_ms": 50, "rtt_ms": 5}, {"p95_ms": 300}, router_node="noda2", node_id="noda2", ) remote_score = self._score( {"wait_ms": 60, "rtt_ms": 80}, {"p95_ms": 350}, router_node="noda2", node_id="noda1", ) self.assertLess(local_score, remote_score, "Local should win when loads similar") def test_remote_wins_when_local_saturated(self): local_score = self._score( {"wait_ms": 2000, "rtt_ms": 5, "mem_pressure": "high"}, {"p95_ms": 1500}, router_node="noda2", node_id="noda2", ) remote_score = self._score( {"wait_ms": 100, "rtt_ms": 80}, {"p95_ms": 400}, router_node="noda2", node_id="noda1", ) self.assertLess(remote_score, local_score, "Remote should win when local is saturated") def test_mem_pressure_high_adds_penalty(self): score_normal = self._score( {"wait_ms": 0}, {}, router_node="noda1", node_id="noda2", ) score_high_mem = self._score( {"wait_ms": 0, "mem_pressure": "high"}, {}, router_node="noda1", node_id="noda2", ) self.assertEqual(score_high_mem - score_normal, 300) def test_no_silent_fallback_contract(self): """Verify scoring never returns negative score for non-local nodes (no accidental preference).""" score = self._score( {"wait_ms": 100, "rtt_ms": 50}, {"p95_ms": 200}, router_node="noda1", node_id="noda2", # noda2 != router's noda1 → no local bonus ) self.assertGreater(score, 0, "Remote node score must be positive") # ═══════════════════════════════════════════════════════════════════════════ # Invariants: no silent fallback, all voice failures log + increment counter # ═══════════════════════════════════════════════════════════════════════════ class TestVoiceMetricsCardinality(unittest.TestCase): """Voice metrics must use bounded label values to prevent cardinality explosion.""" VALID_CAP_LABELS = {"voice_tts", "voice_llm", "voice_stt"} def test_valid_cap_labels_are_bounded(self): """Only 3 valid cap labels exist — prevents Prometheus cardinality explosion.""" self.assertEqual(len(self.VALID_CAP_LABELS), 3) self.assertIn("voice_tts", self.VALID_CAP_LABELS) self.assertIn("voice_llm", self.VALID_CAP_LABELS) self.assertIn("voice_stt", self.VALID_CAP_LABELS) self.assertNotIn("voice_sst", self.VALID_CAP_LABELS) # typo guard def test_no_sst_label_in_router_main(self): """Router main.py must not contain 'voice_sst' (typo guard).""" router_main = os.path.join( os.path.dirname(__file__), "..", "services", "router", "main.py" ) with open(router_main, "r") as f: content = f.read() self.assertNotIn("voice_sst", content, "Typo 'voice_sst' found in router/main.py") def test_no_sst_label_in_worker(self): """Node-worker must not contain 'voice_sst' (typo guard).""" for fname in ["worker.py", "main.py", "config.py"]: path = os.path.join(os.path.dirname(__file__), "..", "services", "node-worker", fname) with open(path, "r") as f: content = f.read() self.assertNotIn("voice_sst", content, f"Typo 'voice_sst' found in {fname}") def test_no_sst_label_in_offload_client(self): """offload_client.py must not contain 'voice.sst' subject.""" path = os.path.join( os.path.dirname(__file__), "..", "services", "router", "offload_client.py" ) with open(path, "r") as f: content = f.read() self.assertNotIn("voice.sst", content, "Typo 'voice.sst' found in offload_client.py") def test_router_valid_caps_set_uses_stt(self): """Router voice endpoint must validate 'stt' not 'sst'.""" router_main = os.path.join( os.path.dirname(__file__), "..", "services", "router", "main.py" ) with open(router_main, "r") as f: content = f.read() self.assertIn('"stt"', content, "Router must include 'stt' in valid_caps") def test_caps_semantics_runtime_separation_in_worker_main(self): """node-worker /caps must separate semantic caps from runtime NATS subscriptions.""" main_path = os.path.join( os.path.dirname(__file__), "..", "services", "node-worker", "main.py" ) with open(main_path, "r") as f: content = f.read() self.assertIn("nats_subscriptions", content, "Operational NATS subscription state must be in runtime section, separate from capabilities") # capabilities must not reference _VOICE_SUBJECTS for semantic flags # (allowed in runtime section only) import re caps_block = re.search(r'"capabilities":\s*\{(.*?)\},', content, re.DOTALL) if caps_block: caps_text = caps_block.group(1) self.assertNotIn("_VOICE_SUBJECTS", caps_text, "Semantic caps must not depend on _VOICE_SUBJECTS (NATS state)") class TestVoiceHANoSilentFallback(unittest.TestCase): def setUp(self): self._worker_path = os.path.join( os.path.dirname(__file__), "..", "services", "node-worker" ) if self._worker_path not in sys.path: sys.path.insert(0, self._worker_path) def test_handle_voice_request_logs_warning_on_busy(self): """_handle_voice_request must log WARNING (not silently skip) when semaphore full.""" import worker # Create a full semaphore (0 slots) full_sem = asyncio.Semaphore(0) msg = MagicMock() msg.data = b'{"job_id":"test","required_type":"tts","payload":{},"deadline_ts":9999999999999}' replies = [] async def _fake_reply(m, resp): replies.append(resp) async def run(): with patch.object(worker, "_reply", side_effect=_fake_reply), \ patch.object(worker, "logger") as mock_log: await worker._handle_voice_request(msg, voice_sem=full_sem, cap_key="voice.tts") return mock_log mock_log = asyncio.run(run()) # Must have sent a TOO_BUSY response self.assertTrue(len(replies) >= 1) resp = replies[0] self.assertIn(resp.status, ("busy", "error")) def test_voice_job_not_silently_ignored(self): """Voice job handler must always send a reply, never drop the message.""" import worker sem = asyncio.Semaphore(2) msg = MagicMock() msg.data = b'invalid json{' # will raise JSONDecodeError replies = [] async def _fake_reply(m, resp): replies.append(resp) async def run(): with patch.object(worker, "_reply", side_effect=_fake_reply): await worker._handle_voice_request(msg, voice_sem=sem, cap_key="voice.tts") asyncio.run(run()) self.assertTrue(len(replies) >= 1, "Must always send a reply, even on parse error") if __name__ == "__main__": unittest.main()