daarion-admin/microdao-daarion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
79db053b38fc7025901d6b049d9065b49eedcc39
microdao-daarion/tests/test_voice_ha.py
Apple 129e4ea1fc feat(platform): add new services, tools, tests and crews modules 
New router intelligence modules (26 files): alert_ingest/store, audit_store,
architecture_pressure, backlog_generator/store, cost_analyzer, data_governance,
dependency_scanner, drift_analyzer, incident_* (5 files), llm_enrichment,
platform_priority_digest, provider_budget, release_check_runner, risk_* (6 files),
signature_state_store, sofiia_auto_router, tool_governance

New services:
- sofiia-console: Dockerfile, adapters/, monitor/nodes/ops/voice modules, launchd, react static
- memory-service: integration_endpoints, integrations, voice_endpoints, static UI
- aurora-service: full app suite (analysis, job_store, orchestrator, reporting, schemas, subagents)
- sofiia-supervisor: new supervisor service
- aistalk-bridge-lite: Telegram bridge lite
- calendar-service: CalDAV calendar service with reminders
- mlx-stt-service / mlx-tts-service: Apple Silicon speech services
- binance-bot-monitor: market monitor service
- node-worker: STT/TTS memory providers

New tools (9): agent_email, browser_tool, contract_tool, observability_tool,
oncall_tool, pr_reviewer_tool, repo_tool, safe_code_executor, secure_vault

New crews: agromatrix_crew (10 modules: depth_classifier, doc_facts, doc_focus,
farm_state, light_reply, llm_factory, memory_manager, proactivity, reflection_engine,
session_context, style_adapter, telemetry)

Tests: 85+ test files for all new modules
Made-with: Cursor
2026-03-03 07:14:14 -08:00

515 lines
22 KiB
Python
Raw Blame History

"""
Voice HA (PR1 + PR2 + PR3) contract tests.
Tests enforce:
1. Node Worker voice capability flags (voice_tts, voice_llm, voice_stt)
2. Voice semaphore isolation from generic concurrency
3. Router voice scoring: prefer local, penalise high load
4. offload_client supports voice.* subject patterns
5. BFF VOICE_HA_ENABLED feature flag routes correctly
"""
import asyncio
import importlib
import importlib.util
import os
import sys
import types
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
# ── helpers ──────────────────────────────────────────────────────────────────
def _reload_config(overrides: dict):
"""Reload node-worker config with env overrides."""
worker_path = os.path.join(os.path.dirname(__file__), "..", "services", "node-worker")
if worker_path not in sys.path:
sys.path.insert(0, worker_path)
for k, v in overrides.items():
os.environ[k] = v
try:
import config as _cfg
importlib.reload(_cfg)
return _cfg
finally:
pass # caller restores env if needed
# ═══════════════════════════════════════════════════════════════════════════
# PR1: Node Worker capability flags
# ═══════════════════════════════════════════════════════════════════════════
class TestNodeWorkerVoiceCaps(unittest.TestCase):
def setUp(self):
self._worker_path = os.path.join(
os.path.dirname(__file__), "..", "services", "node-worker"
)
if self._worker_path not in sys.path:
sys.path.insert(0, self._worker_path)
def test_config_has_voice_concurrency_vars(self):
"""Voice HA concurrency env vars must exist in config."""
cfg = _reload_config({
"STT_PROVIDER": "memory_service",
"TTS_PROVIDER": "memory_service",
"VOICE_MAX_CONCURRENT_TTS": "4",
"VOICE_MAX_CONCURRENT_LLM": "2",
"VOICE_MAX_CONCURRENT_STT": "2",
})
self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 4)
self.assertEqual(cfg.VOICE_MAX_CONCURRENT_LLM, 2)
self.assertEqual(cfg.VOICE_MAX_CONCURRENT_STT, 2)
def test_config_has_voice_deadline_vars(self):
cfg = _reload_config({
"VOICE_TTS_DEADLINE_MS": "3000",
"VOICE_LLM_FAST_MS": "9000",
"VOICE_LLM_QUALITY_MS": "12000",
"VOICE_STT_DEADLINE_MS": "6000",
})
self.assertEqual(cfg.VOICE_TTS_DEADLINE_MS, 3000)
self.assertEqual(cfg.VOICE_LLM_FAST_MS, 9000)
self.assertEqual(cfg.VOICE_STT_DEADLINE_MS, 6000)
def test_voice_tts_false_when_tts_provider_none(self):
"""voice_tts SEMANTIC capability must be False when TTS_PROVIDER=none."""
cfg = _reload_config({"TTS_PROVIDER": "none"})
self.assertEqual(cfg.TTS_PROVIDER, "none")
# Semantic: voice_tts = (TTS_PROVIDER != "none")
voice_tts_cap = cfg.TTS_PROVIDER != "none"
self.assertFalse(voice_tts_cap)
def test_voice_tts_true_when_memory_service(self):
"""voice_tts SEMANTIC capability must be True when TTS_PROVIDER=memory_service."""
cfg = _reload_config({"TTS_PROVIDER": "memory_service"})
self.assertEqual(cfg.TTS_PROVIDER, "memory_service")
# Semantic: provider configured → capability available (not NATS-dependent)
voice_tts_cap = cfg.TTS_PROVIDER != "none"
self.assertTrue(voice_tts_cap)
def test_voice_cap_semantic_not_nats_dependent(self):
"""voice_* capability must NOT depend on NATS subscription state.
Semantics = provider configured.
Operational state (NATS active) is separate (runtime.nats_subscriptions).
"""
cfg = _reload_config({"TTS_PROVIDER": "memory_service", "STT_PROVIDER": "memory_service"})
# Even if NATS subscriptions were empty (reconnect / restart race),
# semantic caps must still be True
voice_tts_semantic = cfg.TTS_PROVIDER != "none"
voice_stt_semantic = cfg.STT_PROVIDER != "none"
self.assertTrue(voice_tts_semantic, "voice_tts semantic must be True regardless of NATS")
self.assertTrue(voice_stt_semantic, "voice_stt semantic must be True regardless of NATS")
def test_voice_concurrency_env_override(self):
cfg = _reload_config({"VOICE_MAX_CONCURRENT_TTS": "8"})
self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 8)
def test_voice_semaphores_independent_from_generic(self):
"""Voice semaphore limit must not equal MAX_CONCURRENCY (they are independent)."""
cfg = _reload_config({
"NODE_WORKER_MAX_CONCURRENCY": "2",
"VOICE_MAX_CONCURRENT_TTS": "4",
})
self.assertEqual(cfg.MAX_CONCURRENCY, 2)
self.assertEqual(cfg.VOICE_MAX_CONCURRENT_TTS, 4)
self.assertNotEqual(cfg.MAX_CONCURRENCY, cfg.VOICE_MAX_CONCURRENT_TTS)
class TestNodeWorkerVoiceSubjects(unittest.TestCase):
"""Worker must register voice.* NATS subjects when providers configured."""
def setUp(self):
self._worker_path = os.path.join(
os.path.dirname(__file__), "..", "services", "node-worker"
)
if self._worker_path not in sys.path:
sys.path.insert(0, self._worker_path)
def test_voice_subjects_set_exists(self):
import worker
self.assertTrue(hasattr(worker, "_VOICE_SUBJECTS"))
self.assertIsInstance(worker._VOICE_SUBJECTS, set)
def test_voice_semaphores_defined(self):
import worker
self.assertTrue(hasattr(worker, "_voice_sem_tts"))
self.assertTrue(hasattr(worker, "_voice_sem_llm"))
self.assertTrue(hasattr(worker, "_voice_sem_stt"))
self.assertIsInstance(worker._voice_sem_tts, asyncio.Semaphore)
def test_voice_semaphore_map_keys(self):
import worker
sem_map = worker._VOICE_SEMAPHORES
self.assertIn("voice.tts", sem_map)
self.assertIn("voice.llm", sem_map)
self.assertIn("voice.stt", sem_map)
class TestFabricMetricsVoice(unittest.TestCase):
"""fabric_metrics must expose voice HA metric helpers."""
def setUp(self):
self._worker_path = os.path.join(
os.path.dirname(__file__), "..", "services", "node-worker"
)
if self._worker_path not in sys.path:
sys.path.insert(0, self._worker_path)
def test_voice_metric_functions_exist(self):
import fabric_metrics as fm
self.assertTrue(callable(fm.inc_voice_job))
self.assertTrue(callable(fm.set_voice_inflight))
self.assertTrue(callable(fm.observe_voice_latency))
def test_voice_metric_functions_no_raise(self):
import fabric_metrics as fm
# Should not raise regardless of prometheus_client availability
fm.inc_voice_job("voice.tts", "ok")
fm.set_voice_inflight("voice.tts", 1)
fm.observe_voice_latency("voice.tts", 1500)
# ═══════════════════════════════════════════════════════════════════════════
# PR2: Router offload_client voice subjects
# ═══════════════════════════════════════════════════════════════════════════
class TestOffloadClientVoiceSubjects(unittest.TestCase):
def setUp(self):
self._router_path = os.path.join(
os.path.dirname(__file__), "..", "services", "router"
)
if self._router_path not in sys.path:
sys.path.insert(0, self._router_path)
def test_offload_voice_tts_uses_dotted_subject(self):
"""voice.tts → node.{id}.voice.tts.request (not node.{id}.tts.request)."""
import offload_client
# The subject is built as f"node.{node_id}.{required_type}.request"
# So required_type="voice.tts" → "node.noda2.voice.tts.request"
node_id = "noda2"
req_type = "voice.tts"
expected_subject = f"node.{node_id}.{req_type}.request"
self.assertEqual(expected_subject, "node.noda2.voice.tts.request")
def test_offload_generic_tts_uses_flat_subject(self):
"""Generic tts → node.{id}.tts.request (unchanged)."""
req_type = "tts"
node_id = "noda2"
expected_subject = f"node.{node_id}.{req_type}.request"
self.assertEqual(expected_subject, "node.noda2.tts.request")
# Must differ from voice subject
self.assertNotEqual("node.noda2.tts.request", "node.noda2.voice.tts.request")
def test_offload_client_accepts_voice_type(self):
"""offload_infer signature must accept voice.* types (no Literal restriction)."""
import inspect
import offload_client
sig = inspect.signature(offload_client.offload_infer)
param = sig.parameters.get("required_type")
self.assertIsNotNone(param, "required_type parameter must exist")
# Must not be restricted to old Literal — annotation should be str or Any
ann = param.annotation
ann_str = str(ann)
self.assertNotIn("Literal", ann_str,
"required_type must not be Literal (voice.* types needed)")
class TestRouterFabricMetricsVoice(unittest.TestCase):
"""Test router/fabric_metrics.py voice helper functions.
Loads the module directly via importlib to avoid collision with
node-worker's fabric_metrics that is earlier in sys.path.
"""
def _load_router_fm(self):
router_path = os.path.join(
os.path.dirname(__file__), "..", "services", "router", "fabric_metrics.py"
)
spec = importlib.util.spec_from_file_location("router_fabric_metrics", router_path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def test_router_voice_metric_functions_exist(self):
fm = self._load_router_fm()
self.assertTrue(callable(fm.inc_voice_cap_request))
self.assertTrue(callable(fm.inc_voice_offload))
self.assertTrue(callable(fm.set_voice_breaker))
self.assertTrue(callable(fm.observe_voice_score))
def test_router_voice_metrics_no_raise(self):
fm = self._load_router_fm()
fm.inc_voice_cap_request("voice_tts", "ok")
fm.inc_voice_offload("voice_tts", "noda2", "ok")
fm.set_voice_breaker("voice_tts", "noda2", False)
fm.observe_voice_score("voice_tts", 250.0)
# ═══════════════════════════════════════════════════════════════════════════
# PR3: BFF config feature flag
# ═══════════════════════════════════════════════════════════════════════════
class TestBFFVoiceHAConfig(unittest.TestCase):
def setUp(self):
self._console_path = os.path.join(
os.path.dirname(__file__), "..", "services", "sofiia-console", "app"
)
# Add parent for package import
parent = os.path.join(os.path.dirname(__file__), "..", "services", "sofiia-console")
if parent not in sys.path:
sys.path.insert(0, parent)
def _reload_bff_config(self, voice_ha: str = "false"):
os.environ["VOICE_HA_ENABLED"] = voice_ha
from app import config as bff_config
importlib.reload(bff_config)
return bff_config
def test_voice_ha_disabled_by_default(self):
os.environ.pop("VOICE_HA_ENABLED", None)
from app import config as bff_config
importlib.reload(bff_config)
self.assertFalse(bff_config.is_voice_ha_enabled())
def test_voice_ha_enabled_via_env(self):
cfg = self._reload_bff_config("true")
self.assertTrue(cfg.is_voice_ha_enabled())
def test_voice_ha_disabled_explicit(self):
cfg = self._reload_bff_config("false")
self.assertFalse(cfg.is_voice_ha_enabled())
def test_voice_ha_enabled_variants(self):
for val in ("1", "yes", "true", "True", "YES"):
os.environ["VOICE_HA_ENABLED"] = val
from app import config as bff_config
importlib.reload(bff_config)
self.assertTrue(bff_config.is_voice_ha_enabled(), f"Failed for value: {val}")
def test_voice_ha_router_url_default(self):
os.environ.pop("VOICE_HA_ROUTER_URL", None)
os.environ.pop("NODES_NODA2_ROUTER_URL", None)
from app import config as bff_config
importlib.reload(bff_config)
url = bff_config.get_voice_ha_router_url()
# Must return a string URL, not empty
self.assertIsInstance(url, str)
self.assertTrue(url.startswith("http"))
def test_voice_ha_router_url_override(self):
os.environ["VOICE_HA_ROUTER_URL"] = "http://router-ha:9200"
from app import config as bff_config
importlib.reload(bff_config)
url = bff_config.get_voice_ha_router_url()
self.assertEqual(url, "http://router-ha:9200")
del os.environ["VOICE_HA_ROUTER_URL"]
def tearDown(self):
os.environ.pop("VOICE_HA_ENABLED", None)
os.environ.pop("VOICE_HA_ROUTER_URL", None)
# ═══════════════════════════════════════════════════════════════════════════
# Voice scoring logic: prefer local, penalise high load
# ═══════════════════════════════════════════════════════════════════════════
class TestVoiceHAScoringLogic(unittest.TestCase):
"""Unit tests for voice node scoring formula (inline, no Router import needed)."""
def _score(self, node_load: dict, runtime_load: dict,
router_node: str, node_id: str,
prefer_bonus: int = 200) -> int:
"""Replicate the scoring formula from Router voice_capability_offload."""
nl = node_load
rl = runtime_load
wait_ms = nl.get("wait_ms", 0) or nl.get("inflight", 0) * 50
rtt_ms = nl.get("rtt_ms", 0)
p95_ms = rl.get("p95_ms", 0) if rl else 0
mem_penalty = 300 if nl.get("mem_pressure") == "high" else 0
local_bonus = prefer_bonus if node_id.lower() == router_node.lower() else 0
return wait_ms + rtt_ms + p95_ms + mem_penalty - local_bonus
def test_local_node_preferred_when_load_similar(self):
local_score = self._score(
{"wait_ms": 50, "rtt_ms": 5},
{"p95_ms": 300},
router_node="noda2", node_id="noda2",
)
remote_score = self._score(
{"wait_ms": 60, "rtt_ms": 80},
{"p95_ms": 350},
router_node="noda2", node_id="noda1",
)
self.assertLess(local_score, remote_score, "Local should win when loads similar")
def test_remote_wins_when_local_saturated(self):
local_score = self._score(
{"wait_ms": 2000, "rtt_ms": 5, "mem_pressure": "high"},
{"p95_ms": 1500},
router_node="noda2", node_id="noda2",
)
remote_score = self._score(
{"wait_ms": 100, "rtt_ms": 80},
{"p95_ms": 400},
router_node="noda2", node_id="noda1",
)
self.assertLess(remote_score, local_score, "Remote should win when local is saturated")
def test_mem_pressure_high_adds_penalty(self):
score_normal = self._score(
{"wait_ms": 0},
{},
router_node="noda1", node_id="noda2",
)
score_high_mem = self._score(
{"wait_ms": 0, "mem_pressure": "high"},
{},
router_node="noda1", node_id="noda2",
)
self.assertEqual(score_high_mem - score_normal, 300)
def test_no_silent_fallback_contract(self):
"""Verify scoring never returns negative score for non-local nodes (no accidental preference)."""
score = self._score(
{"wait_ms": 100, "rtt_ms": 50},
{"p95_ms": 200},
router_node="noda1", node_id="noda2", # noda2 != router's noda1 → no local bonus
)
self.assertGreater(score, 0, "Remote node score must be positive")
# ═══════════════════════════════════════════════════════════════════════════
# Invariants: no silent fallback, all voice failures log + increment counter
# ═══════════════════════════════════════════════════════════════════════════
class TestVoiceMetricsCardinality(unittest.TestCase):
"""Voice metrics must use bounded label values to prevent cardinality explosion."""
VALID_CAP_LABELS = {"voice_tts", "voice_llm", "voice_stt"}
def test_valid_cap_labels_are_bounded(self):
"""Only 3 valid cap labels exist — prevents Prometheus cardinality explosion."""
self.assertEqual(len(self.VALID_CAP_LABELS), 3)
self.assertIn("voice_tts", self.VALID_CAP_LABELS)
self.assertIn("voice_llm", self.VALID_CAP_LABELS)
self.assertIn("voice_stt", self.VALID_CAP_LABELS)
self.assertNotIn("voice_sst", self.VALID_CAP_LABELS) # typo guard
def test_no_sst_label_in_router_main(self):
"""Router main.py must not contain 'voice_sst' (typo guard)."""
router_main = os.path.join(
os.path.dirname(__file__), "..", "services", "router", "main.py"
)
with open(router_main, "r") as f:
content = f.read()
self.assertNotIn("voice_sst", content, "Typo 'voice_sst' found in router/main.py")
def test_no_sst_label_in_worker(self):
"""Node-worker must not contain 'voice_sst' (typo guard)."""
for fname in ["worker.py", "main.py", "config.py"]:
path = os.path.join(os.path.dirname(__file__), "..", "services", "node-worker", fname)
with open(path, "r") as f:
content = f.read()
self.assertNotIn("voice_sst", content, f"Typo 'voice_sst' found in {fname}")
def test_no_sst_label_in_offload_client(self):
"""offload_client.py must not contain 'voice.sst' subject."""
path = os.path.join(
os.path.dirname(__file__), "..", "services", "router", "offload_client.py"
)
with open(path, "r") as f:
content = f.read()
self.assertNotIn("voice.sst", content, "Typo 'voice.sst' found in offload_client.py")
def test_router_valid_caps_set_uses_stt(self):
"""Router voice endpoint must validate 'stt' not 'sst'."""
router_main = os.path.join(
os.path.dirname(__file__), "..", "services", "router", "main.py"
)
with open(router_main, "r") as f:
content = f.read()
self.assertIn('"stt"', content, "Router must include 'stt' in valid_caps")
def test_caps_semantics_runtime_separation_in_worker_main(self):
"""node-worker /caps must separate semantic caps from runtime NATS subscriptions."""
main_path = os.path.join(
os.path.dirname(__file__), "..", "services", "node-worker", "main.py"
)
with open(main_path, "r") as f:
content = f.read()
self.assertIn("nats_subscriptions", content,
"Operational NATS subscription state must be in runtime section, separate from capabilities")
# capabilities must not reference _VOICE_SUBJECTS for semantic flags
# (allowed in runtime section only)
import re
caps_block = re.search(r'"capabilities":\s*\{(.*?)\},', content, re.DOTALL)
if caps_block:
caps_text = caps_block.group(1)
self.assertNotIn("_VOICE_SUBJECTS", caps_text,
"Semantic caps must not depend on _VOICE_SUBJECTS (NATS state)")
class TestVoiceHANoSilentFallback(unittest.TestCase):
def setUp(self):
self._worker_path = os.path.join(
os.path.dirname(__file__), "..", "services", "node-worker"
)
if self._worker_path not in sys.path:
sys.path.insert(0, self._worker_path)
def test_handle_voice_request_logs_warning_on_busy(self):
"""_handle_voice_request must log WARNING (not silently skip) when semaphore full."""
import worker
# Create a full semaphore (0 slots)
full_sem = asyncio.Semaphore(0)
msg = MagicMock()
msg.data = b'{"job_id":"test","required_type":"tts","payload":{},"deadline_ts":9999999999999}'
replies = []
async def _fake_reply(m, resp):
replies.append(resp)
async def run():
with patch.object(worker, "_reply", side_effect=_fake_reply), \
patch.object(worker, "logger") as mock_log:
await worker._handle_voice_request(msg, voice_sem=full_sem, cap_key="voice.tts")
return mock_log
mock_log = asyncio.run(run())
# Must have sent a TOO_BUSY response
self.assertTrue(len(replies) >= 1)
resp = replies[0]
self.assertIn(resp.status, ("busy", "error"))
def test_voice_job_not_silently_ignored(self):
"""Voice job handler must always send a reply, never drop the message."""
import worker
sem = asyncio.Semaphore(2)
msg = MagicMock()
msg.data = b'invalid json{' # will raise JSONDecodeError
replies = []
async def _fake_reply(m, resp):
replies.append(resp)
async def run():
with patch.object(worker, "_reply", side_effect=_fake_reply):
await worker._handle_voice_request(msg, voice_sem=sem, cap_key="voice.tts")
asyncio.run(run())
self.assertTrue(len(replies) >= 1, "Must always send a reply, even on parse error")
if __name__ == "__main__":
unittest.main()
Reference in New Issue View Git Blame Copy Permalink