129e4ea1fc
New router intelligence modules (26 files): alert_ingest/store, audit_store, architecture_pressure, backlog_generator/store, cost_analyzer, data_governance, dependency_scanner, drift_analyzer, incident_* (5 files), llm_enrichment, platform_priority_digest, provider_budget, release_check_runner, risk_* (6 files), signature_state_store, sofiia_auto_router, tool_governance New services: - sofiia-console: Dockerfile, adapters/, monitor/nodes/ops/voice modules, launchd, react static - memory-service: integration_endpoints, integrations, voice_endpoints, static UI - aurora-service: full app suite (analysis, job_store, orchestrator, reporting, schemas, subagents) - sofiia-supervisor: new supervisor service - aistalk-bridge-lite: Telegram bridge lite - calendar-service: CalDAV calendar service with reminders - mlx-stt-service / mlx-tts-service: Apple Silicon speech services - binance-bot-monitor: market monitor service - node-worker: STT/TTS memory providers New tools (9): agent_email, browser_tool, contract_tool, observability_tool, oncall_tool, pr_reviewer_tool, repo_tool, safe_code_executor, secure_vault New crews: agromatrix_crew (10 modules: depth_classifier, doc_facts, doc_focus, farm_state, light_reply, llm_factory, memory_manager, proactivity, reflection_engine, session_context, style_adapter, telemetry) Tests: 85+ test files for all new modules Made-with: Cursor
227 lines
8.5 KiB
Python
227 lines
8.5 KiB
Python
"""
|
|
tests/test_release_check_risk_delta_watch.py — Unit tests for risk_delta_watch gate.
|
|
|
|
Tests:
|
|
- warn mode: gate passes with recommendations when delta >= warn_delta
|
|
- strict mode: should_fail=True for p0_services when delta >= fail_delta
|
|
- missing history → skipped (non-fatal)
|
|
- tool error → skipped (non-fatal)
|
|
"""
|
|
import asyncio
|
|
import datetime
|
|
import sys
|
|
import pytest
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "services" / "router"))
|
|
|
|
from release_check_runner import _run_risk_delta_watch
|
|
from risk_history_store import MemoryRiskHistoryStore, RiskSnapshot, set_risk_history_store
|
|
|
|
|
|
def _snap(service, env, score, hours_ago=0) -> RiskSnapshot:
|
|
ts = (datetime.datetime.utcnow() - datetime.timedelta(hours=hours_ago)).isoformat()
|
|
return RiskSnapshot(ts=ts, service=service, env=env, score=score, band="medium")
|
|
|
|
|
|
def _make_tm(score: int, service: str = "gateway", fail_execute: bool = False):
|
|
"""Stub ToolManager returning a fixed risk score."""
|
|
tm = MagicMock()
|
|
if fail_execute:
|
|
tm.execute_tool = AsyncMock(side_effect=RuntimeError("timeout"))
|
|
else:
|
|
result = MagicMock()
|
|
result.success = True
|
|
result.error = None
|
|
result.result = {"service": service, "env": "prod", "score": score, "band": "medium"}
|
|
tm.execute_tool = AsyncMock(return_value=result)
|
|
return tm
|
|
|
|
|
|
# ─── Warn mode ────────────────────────────────────────────────────────────────
|
|
|
|
class TestRiskDeltaWarnMode:
|
|
def _run(self, **kwargs):
|
|
return asyncio.run(_run_risk_delta_watch(**kwargs))
|
|
|
|
def test_delta_below_warn_is_clean(self, tmp_path):
|
|
"""delta 5 < warn 10 → no recommendations."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("gateway", "prod", 50, hours_ago=25),
|
|
_snap("gateway", "prod", 55, hours_ago=1), # delta=5
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=55)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert ok is True
|
|
assert gate["status"] == "pass"
|
|
assert not gate.get("skipped")
|
|
assert gate.get("delta") == 5
|
|
assert gate.get("regression_warn") is False
|
|
assert gate.get("recommendations", []) == []
|
|
|
|
def test_delta_at_warn_threshold_adds_recommendation(self, tmp_path):
|
|
"""delta 10 == warn_delta 10 → warn=True, rec added."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("gateway", "prod", 40, hours_ago=25),
|
|
_snap("gateway", "prod", 50, hours_ago=1), # delta=10
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=50)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert ok is True
|
|
assert gate["regression_warn"] is True
|
|
assert len(gate.get("recommendations", [])) > 0
|
|
|
|
def test_delta_above_fail_adds_strong_recommendation(self, tmp_path):
|
|
"""delta 25 >= fail 20 → regression_fail=True, recommendations urgent."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("gateway", "prod", 30, hours_ago=25),
|
|
_snap("gateway", "prod", 55, hours_ago=1), # delta=25
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=55)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert ok is True # gate helper always returns ok=True
|
|
assert gate["regression_fail"] is True
|
|
assert any("FAIL" in r or "fail" in r.lower() for r in gate.get("recommendations", []))
|
|
|
|
|
|
# ─── Strict mode: should_fail for p0_services ────────────────────────────────
|
|
|
|
class TestRiskDeltaStrictMode:
|
|
def _run(self, **kwargs):
|
|
return asyncio.run(_run_risk_delta_watch(**kwargs))
|
|
|
|
def test_should_fail_set_for_p0_service_with_high_delta(self, tmp_path):
|
|
"""gateway is p0, delta 25 >= fail 20 → should_fail=True."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("gateway", "prod", 30, hours_ago=25),
|
|
_snap("gateway", "prod", 55, hours_ago=1),
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=55, service="gateway")
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert gate["should_fail"] is True
|
|
assert gate["is_p0"] is True
|
|
|
|
def test_should_fail_false_for_non_p0_service(self, tmp_path):
|
|
"""memory-service is not p0 → should_fail=False even if delta >= fail."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("memory-service", "prod", 10, hours_ago=25),
|
|
_snap("memory-service", "prod", 40, hours_ago=1), # delta=30
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=40, service="memory-service")
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="memory-service", env="prod",
|
|
)
|
|
assert gate["should_fail"] is False
|
|
assert gate["is_p0"] is False
|
|
|
|
def test_custom_fail_delta_respected(self, tmp_path):
|
|
"""Override fail_delta=30; delta 25 < 30 → should_fail=False."""
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([
|
|
_snap("gateway", "prod", 30, hours_ago=25),
|
|
_snap("gateway", "prod", 55, hours_ago=1), # delta=25
|
|
])
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=55, service="gateway")
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
fail_delta=30,
|
|
)
|
|
assert gate["should_fail"] is False # 25 < 30
|
|
|
|
def test_effective_thresholds_in_gate(self, tmp_path):
|
|
store = MemoryRiskHistoryStore()
|
|
store.write_snapshot([_snap("gateway", "prod", 40, hours_ago=25),
|
|
_snap("gateway", "prod", 60, hours_ago=1)])
|
|
set_risk_history_store(store)
|
|
tm = _make_tm(score=60)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
warn_delta=5, fail_delta=15,
|
|
)
|
|
assert gate["effective_warn_delta"] == 5
|
|
assert gate["effective_fail_delta"] == 15
|
|
|
|
|
|
# ─── Non-fatal (skipped) ──────────────────────────────────────────────────────
|
|
|
|
class TestRiskDeltaNonFatal:
|
|
def _run(self, **kwargs):
|
|
return asyncio.run(_run_risk_delta_watch(**kwargs))
|
|
|
|
def test_no_history_skips_gracefully(self, tmp_path):
|
|
"""Empty history store → skipped=True, ok=True, no crash."""
|
|
store = MemoryRiskHistoryStore()
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=60)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert ok is True
|
|
assert gate.get("skipped") is True
|
|
assert gate["status"] == "pass"
|
|
assert any("baseline" in r.lower() or "history" in r.lower()
|
|
for r in gate.get("recommendations", []))
|
|
|
|
def test_tool_error_skips_gracefully(self, tmp_path):
|
|
"""risk_engine_tool raises → skipped, never blocks."""
|
|
store = MemoryRiskHistoryStore()
|
|
set_risk_history_store(store)
|
|
|
|
tm = _make_tm(score=0, fail_execute=True)
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="gateway", env="prod",
|
|
)
|
|
assert ok is True
|
|
assert gate.get("skipped") is True
|
|
|
|
def test_no_service_name_skips(self, tmp_path):
|
|
"""Empty service_name → skipped immediately."""
|
|
store = MemoryRiskHistoryStore()
|
|
set_risk_history_store(store)
|
|
tm = MagicMock()
|
|
tm.execute_tool = AsyncMock()
|
|
|
|
ok, gate = self._run(
|
|
tool_manager=tm, agent_id="ops",
|
|
service_name="", env="prod",
|
|
)
|
|
assert ok is True
|
|
assert gate.get("skipped") is True
|
|
tm.execute_tool.assert_not_called()
|