feat(sofiia-console): add auto-evidence and post-review generation from runbook runs
- adds runbook_artifacts.py: server-side render of release_evidence.md and
post_review.md from DB step results (no shell); saves to
SOFIIA_DATA_DIR/release_artifacts/<run_id>/
- evidence: auto-fills preflight/smoke/script outcomes, step table, timestamps
- post_review: auto-fills metadata, smoke results, incidents from step statuses;
leaves [TODO] markers for manual observation sections
- adds POST /api/runbooks/runs/{run_id}/evidence and /post_review endpoints
- updates runbook_runs.evidence_path in DB after render
- adds 11 tests covering file creation, key sections, TODO markers, 404s, API
Made-with: Cursor
This commit is contained in:
Apple
249
tests/test_sofiia_runbook_artifacts.py
Normal file
249
tests/test_sofiia_runbook_artifacts.py
Normal file
@@ -0,0 +1,249 @@
|
||||
"""
|
||||
Tests for runbook artifacts renderer (PR4):
|
||||
- render_release_evidence: creates file, contains key sections
|
||||
- render_post_review: creates file, contains TODO markers + auto-filled fields
|
||||
- API endpoints: POST /evidence, /post_review return 200 with path/bytes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_rehearsal_docs(tmp_path):
|
||||
"""tmp_path/docs/runbook with rehearsal checklist."""
|
||||
docs_root = tmp_path / "docs"
|
||||
runbook_dir = docs_root / "runbook"
|
||||
runbook_dir.mkdir(parents=True)
|
||||
(runbook_dir / "rehearsal-v1-30min-checklist.md").write_text(
|
||||
"# Rehearsal v1 — 30-minute execution plan\n\n"
|
||||
"## 0–5 min — Preflight\n\nRun preflight.\n\n"
|
||||
"## 5–10 min — Restart\n\nRestart service.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return docs_root
|
||||
|
||||
|
||||
def _fake_http_proc():
|
||||
"""Fake subprocess for script steps: exit_code=0, stdout=PASS."""
|
||||
from unittest.mock import MagicMock
|
||||
proc = MagicMock()
|
||||
proc.returncode = 0
|
||||
proc.kill = MagicMock()
|
||||
proc.wait = AsyncMock(return_value=None)
|
||||
|
||||
async def _communicate():
|
||||
return b"PASS\n", b""
|
||||
|
||||
proc.communicate = _communicate
|
||||
return proc
|
||||
|
||||
|
||||
async def _create_and_run_all(tmp_path, tmp_docs):
|
||||
"""Create run, execute all auto steps (http_check + script via mocks), return run_id."""
|
||||
import app.docs_index as docs_index_mod
|
||||
import app.runbook_runner as runner_mod
|
||||
import app.db as db_mod
|
||||
import os
|
||||
|
||||
os.environ["SOFIIA_DATA_DIR"] = str(tmp_path / "sofiia-data")
|
||||
await docs_index_mod.rebuild_index(tmp_docs)
|
||||
|
||||
out = await runner_mod.create_run(
|
||||
"runbook/rehearsal-v1-30min-checklist.md",
|
||||
operator_id="test-op",
|
||||
node_id="NODA2",
|
||||
sofiia_url="http://127.0.0.1:8002",
|
||||
)
|
||||
run_id = out["run_id"]
|
||||
steps_total = out["steps_total"]
|
||||
|
||||
# Execute all auto steps (http_check + script)
|
||||
with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_http:
|
||||
async def fake_get(url, **kwargs):
|
||||
if "audit" in str(url):
|
||||
return type("R", (), {"status_code": 401})()
|
||||
return type("R", (), {"status_code": 200})()
|
||||
mock_http.side_effect = fake_get
|
||||
|
||||
with patch("asyncio.create_subprocess_exec", new_callable=AsyncMock) as mock_exec:
|
||||
mock_exec.return_value = _fake_http_proc()
|
||||
|
||||
for _ in range(steps_total):
|
||||
step = await runner_mod.next_step(run_id, operator_id="test-op")
|
||||
if step is None:
|
||||
break
|
||||
if step.get("type") == "manual":
|
||||
await runner_mod.complete_step(run_id, step["step_index"], status="ok", notes="done")
|
||||
|
||||
return run_id
|
||||
|
||||
|
||||
# ── Unit tests: renderer ──────────────────────────────────────────────────────
|
||||
|
||||
def test_render_release_evidence_file_created(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""render_release_evidence creates a file at the expected path."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_release_evidence(run_id))
|
||||
|
||||
assert "evidence_path" in out
|
||||
assert "bytes" in out
|
||||
assert out["bytes"] > 0
|
||||
path = Path(out["evidence_path"])
|
||||
assert path.exists(), f"Expected file at {path}"
|
||||
|
||||
|
||||
def test_render_release_evidence_contains_key_sections(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""Evidence file contains Release metadata, Preflight, Smoke, and Sign-off sections."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_release_evidence(run_id))
|
||||
|
||||
content = Path(out["evidence_path"]).read_text(encoding="utf-8")
|
||||
assert "Release metadata" in content
|
||||
assert "Preflight" in content
|
||||
assert "Smoke" in content
|
||||
assert "Sign-off" in content
|
||||
assert run_id in content
|
||||
|
||||
|
||||
def test_render_release_evidence_includes_step_table(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""Evidence contains a steps summary table with action_types."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_release_evidence(run_id))
|
||||
|
||||
content = Path(out["evidence_path"]).read_text(encoding="utf-8")
|
||||
assert "http_check" in content or "script" in content or "manual" in content
|
||||
assert "All steps summary" in content
|
||||
|
||||
|
||||
def test_render_post_review_file_created(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""render_post_review creates post_review.md."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_post_review(run_id))
|
||||
|
||||
assert "path" in out
|
||||
path = Path(out["path"])
|
||||
assert path.exists()
|
||||
assert out["bytes"] > 0
|
||||
|
||||
|
||||
def test_render_post_review_has_todo_markers(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""Post-review has [TODO] markers for fields requiring manual input."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_post_review(run_id))
|
||||
|
||||
content = Path(out["path"]).read_text(encoding="utf-8")
|
||||
assert "[TODO]" in content
|
||||
assert "Release Metadata" in content
|
||||
assert "Preflight Outcome" in content
|
||||
assert "Smoke Results" in content
|
||||
assert "Action Items" in content
|
||||
|
||||
|
||||
def test_render_post_review_autofills_operator(sofiia_module, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""Post-review auto-fills operator and run_id from DB."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
out = loop.run_until_complete(art_mod.render_post_review(run_id))
|
||||
|
||||
content = Path(out["path"]).read_text(encoding="utf-8")
|
||||
assert "test-op" in content
|
||||
assert run_id in content
|
||||
|
||||
|
||||
def test_render_evidence_404_for_unknown_run(sofiia_module, tmp_path, monkeypatch):
|
||||
"""render_release_evidence raises ValueError for unknown run_id."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
import app.db as _db
|
||||
|
||||
async def run():
|
||||
await _db.init_db()
|
||||
return await art_mod.render_release_evidence("00000000-0000-0000-0000-000000000000")
|
||||
|
||||
with pytest.raises(ValueError, match="Run not found"):
|
||||
loop.run_until_complete(run())
|
||||
|
||||
|
||||
# ── API endpoint tests ────────────────────────────────────────────────────────
|
||||
|
||||
def test_evidence_endpoint_404_for_unknown_run(sofiia_client):
|
||||
"""POST /evidence returns 404 for unknown run_id."""
|
||||
r = sofiia_client.post("/api/runbooks/runs/00000000-0000-0000-0000-000000000000/evidence")
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_post_review_endpoint_404_for_unknown_run(sofiia_client):
|
||||
"""POST /post_review returns 404 for unknown run_id."""
|
||||
r = sofiia_client.post("/api/runbooks/runs/00000000-0000-0000-0000-000000000000/post_review")
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_evidence_endpoint_success(sofiia_module, sofiia_client, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""POST /evidence returns 200 with evidence_path and bytes."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/evidence")
|
||||
assert r.status_code == 200, r.text
|
||||
data = r.json()
|
||||
assert "evidence_path" in data
|
||||
assert data.get("bytes", 0) > 0
|
||||
assert run_id in data.get("evidence_path", "")
|
||||
|
||||
|
||||
def test_post_review_endpoint_success(sofiia_module, sofiia_client, tmp_path, tmp_rehearsal_docs, monkeypatch):
|
||||
"""POST /post_review returns 200 with path and bytes."""
|
||||
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
import app.runbook_artifacts as art_mod
|
||||
|
||||
run_id = loop.run_until_complete(_create_and_run_all(tmp_path, tmp_rehearsal_docs))
|
||||
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/post_review")
|
||||
assert r.status_code == 200, r.text
|
||||
data = r.json()
|
||||
assert "path" in data
|
||||
assert data.get("bytes", 0) > 0
|
||||
Reference in New Issue
Block a user