feat(sofiia-console): add guided runbook runner with http checks and audit integration
adds runbook_runs/runbook_steps state machine parses markdown runbooks into guided steps supports allowlisted http_check (health/metrics/audit) integrates runbook execution with audit trail exposes authenticated runbook runs API Made-with: Cursor
This commit is contained in:
@@ -379,6 +379,40 @@ CREATE TABLE IF NOT EXISTS docs_index_meta (
|
|||||||
value TEXT NOT NULL DEFAULT ''
|
value TEXT NOT NULL DEFAULT ''
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- ── Runbook Runner (PR2) ───────────────────────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS runbook_runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
runbook_path TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'running',
|
||||||
|
current_step INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at REAL NOT NULL,
|
||||||
|
started_at REAL,
|
||||||
|
finished_at REAL,
|
||||||
|
operator_id TEXT,
|
||||||
|
node_id TEXT,
|
||||||
|
sofiia_url TEXT,
|
||||||
|
data_json TEXT,
|
||||||
|
evidence_path TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_runbook_runs_status_created ON runbook_runs(status, created_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_runbook_runs_path_created ON runbook_runs(runbook_path, created_at);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS runbook_steps (
|
||||||
|
run_id TEXT NOT NULL,
|
||||||
|
step_index INTEGER NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
section TEXT,
|
||||||
|
action_type TEXT NOT NULL,
|
||||||
|
action_json TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
result_json TEXT,
|
||||||
|
started_at REAL,
|
||||||
|
finished_at REAL,
|
||||||
|
PRIMARY KEY (run_id, step_index),
|
||||||
|
FOREIGN KEY (run_id) REFERENCES runbook_runs(run_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_runbook_steps_run_status ON runbook_steps(run_id, status);
|
||||||
|
|
||||||
-- ── Graph Intelligence (Hygiene + Reflection) ──────────────────────────────
|
-- ── Graph Intelligence (Hygiene + Reflection) ──────────────────────────────
|
||||||
-- These ADD COLUMN statements are idempotent (IF NOT EXISTS requires SQLite 3.37+).
|
-- These ADD COLUMN statements are idempotent (IF NOT EXISTS requires SQLite 3.37+).
|
||||||
-- On older SQLite they fail silently — init_db() wraps them in a separate try block.
|
-- On older SQLite they fail silently — init_db() wraps them in a separate try block.
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ from .monitor import collect_all_nodes
|
|||||||
from .ops import run_ops_action, OPS_ACTIONS
|
from .ops import run_ops_action, OPS_ACTIONS
|
||||||
from .docs_router import docs_router
|
from .docs_router import docs_router
|
||||||
from .runbooks_router import runbooks_router
|
from .runbooks_router import runbooks_router
|
||||||
|
from .runbook_runs_router import runbook_runs_router
|
||||||
from . import db as _app_db
|
from . import db as _app_db
|
||||||
from .metrics import (
|
from .metrics import (
|
||||||
SOFIIA_SEND_REQUESTS_TOTAL,
|
SOFIIA_SEND_REQUESTS_TOTAL,
|
||||||
@@ -465,6 +466,8 @@ app.add_middleware(
|
|||||||
app.include_router(docs_router)
|
app.include_router(docs_router)
|
||||||
# Runbooks / docs index (read-only search & preview, PR1.1)
|
# Runbooks / docs index (read-only search & preview, PR1.1)
|
||||||
app.include_router(runbooks_router)
|
app.include_router(runbooks_router)
|
||||||
|
# Runbook runs (guided runner, PR2)
|
||||||
|
app.include_router(runbook_runs_router)
|
||||||
|
|
||||||
# ── WebSocket event bus ───────────────────────────────────────────────────────
|
# ── WebSocket event bus ───────────────────────────────────────────────────────
|
||||||
_ws_clients: Set[WebSocket] = set()
|
_ws_clients: Set[WebSocket] = set()
|
||||||
|
|||||||
127
services/sofiia-console/app/runbook_parser.py
Normal file
127
services/sofiia-console/app/runbook_parser.py
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
"""
|
||||||
|
Runbook parser — markdown → list of steps (manual / http_check).
|
||||||
|
PR2: sections by H2 → manual steps; for rehearsal-v1 prepend 3 http_check steps.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RunbookStep:
|
||||||
|
step_index: int
|
||||||
|
title: str
|
||||||
|
section: str | None
|
||||||
|
action_type: str # manual | http_check
|
||||||
|
action_json: Dict[str, Any]
|
||||||
|
status: str = "pending"
|
||||||
|
|
||||||
|
|
||||||
|
def _rehearsal_http_check_steps(base_url: str) -> List[RunbookStep]:
|
||||||
|
"""Three built-in http_check steps for rehearsal v1."""
|
||||||
|
base = (base_url or "http://127.0.0.1:8002").rstrip("/")
|
||||||
|
return [
|
||||||
|
RunbookStep(
|
||||||
|
step_index=0,
|
||||||
|
title="Health check",
|
||||||
|
section="0–5 min — Preflight",
|
||||||
|
action_type="http_check",
|
||||||
|
action_json={
|
||||||
|
"method": "GET",
|
||||||
|
"url_path": "/api/health",
|
||||||
|
"expect": {"status": [200]},
|
||||||
|
"headers": {},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
RunbookStep(
|
||||||
|
step_index=1,
|
||||||
|
title="Metrics check",
|
||||||
|
section="0–5 min — Preflight",
|
||||||
|
action_type="http_check",
|
||||||
|
action_json={
|
||||||
|
"method": "GET",
|
||||||
|
"url_path": "/metrics",
|
||||||
|
"expect": {"status": [200]},
|
||||||
|
"headers": {},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
RunbookStep(
|
||||||
|
step_index=2,
|
||||||
|
title="Audit auth (expect 401 without key)",
|
||||||
|
section="10–15 min — Smoke",
|
||||||
|
action_type="http_check",
|
||||||
|
action_json={
|
||||||
|
"method": "GET",
|
||||||
|
"url_path": "/api/audit",
|
||||||
|
"expect": {"status": [401]},
|
||||||
|
"headers": {},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_sections(markdown: str) -> List[tuple]:
|
||||||
|
"""Split by ## or ###, return [(section_title, content), ...]."""
|
||||||
|
sections: List[tuple] = []
|
||||||
|
current_title = ""
|
||||||
|
current_lines: List[str] = []
|
||||||
|
for line in markdown.splitlines():
|
||||||
|
if re.match(r"^##\s+", line):
|
||||||
|
if current_lines:
|
||||||
|
sections.append((current_title, "\n".join(current_lines).strip()))
|
||||||
|
current_title = line.lstrip("#").strip()[:300]
|
||||||
|
current_lines = []
|
||||||
|
elif re.match(r"^###\s+", line):
|
||||||
|
if current_lines:
|
||||||
|
sections.append((current_title, "\n".join(current_lines).strip()))
|
||||||
|
current_title = line.lstrip("#").strip()[:300]
|
||||||
|
current_lines = []
|
||||||
|
else:
|
||||||
|
current_lines.append(line)
|
||||||
|
if current_lines:
|
||||||
|
sections.append((current_title, "\n".join(current_lines).strip()))
|
||||||
|
return sections
|
||||||
|
|
||||||
|
|
||||||
|
def parse_runbook(
|
||||||
|
runbook_path: str,
|
||||||
|
markdown: str,
|
||||||
|
sofiia_url: str = "http://127.0.0.1:8002",
|
||||||
|
) -> List[RunbookStep]:
|
||||||
|
"""
|
||||||
|
Parse markdown into steps. For rehearsal-v1 prepend 3 http_check steps;
|
||||||
|
rest are manual (one per H2 section with instructions_md).
|
||||||
|
"""
|
||||||
|
path_lower = runbook_path.lower()
|
||||||
|
steps: List[RunbookStep] = []
|
||||||
|
offset = 0
|
||||||
|
|
||||||
|
if "rehearsal" in path_lower and "30min" in path_lower:
|
||||||
|
builtin = _rehearsal_http_check_steps(sofiia_url)
|
||||||
|
steps.extend(builtin)
|
||||||
|
offset = len(builtin)
|
||||||
|
|
||||||
|
sections = _parse_sections(markdown)
|
||||||
|
for i, (title, content) in enumerate(sections):
|
||||||
|
if not title and not content.strip():
|
||||||
|
continue
|
||||||
|
title_clean = title or f"Step {i + 1}"
|
||||||
|
# First section might be title-only; still add as manual
|
||||||
|
instructions = content.strip() or title_clean
|
||||||
|
if len(instructions) > 8000:
|
||||||
|
instructions = instructions[:8000] + "\n..."
|
||||||
|
steps.append(
|
||||||
|
RunbookStep(
|
||||||
|
step_index=offset + i,
|
||||||
|
title=title_clean[:200],
|
||||||
|
section=title_clean[:200] if title else None,
|
||||||
|
action_type="manual",
|
||||||
|
action_json={"instructions_md": instructions, "accept_input": True},
|
||||||
|
status="pending",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return steps
|
||||||
337
services/sofiia-console/app/runbook_runner.py
Normal file
337
services/sofiia-console/app/runbook_runner.py
Normal file
@@ -0,0 +1,337 @@
|
|||||||
|
"""
|
||||||
|
Runbook runner — create run, next_step (execute http_check or return manual), complete_step, abort.
|
||||||
|
PR2: guided execution, allowlisted HTTP only; audit integration.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from . import db as _db
|
||||||
|
from . import docs_store as _docs_store
|
||||||
|
from .audit import audit_log, AuditEvent
|
||||||
|
from .runbook_parser import RunbookStep, parse_runbook
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_ALLOWED_HTTP_PATHS = {"/api/health", "/metrics", "/api/audit"}
|
||||||
|
|
||||||
|
|
||||||
|
def _now_ts() -> float:
|
||||||
|
return time.time()
|
||||||
|
|
||||||
|
|
||||||
|
async def create_run(
|
||||||
|
runbook_path: str,
|
||||||
|
operator_id: str = "",
|
||||||
|
node_id: Optional[str] = None,
|
||||||
|
sofiia_url: Optional[str] = None,
|
||||||
|
data_json: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Parse runbook, insert run + steps, audit. Returns run_id, status, current_step, steps_total."""
|
||||||
|
raw = await _docs_store.get_raw(runbook_path)
|
||||||
|
if not raw:
|
||||||
|
raise ValueError(f"Runbook not found: {runbook_path}")
|
||||||
|
base_url = sofiia_url or "http://127.0.0.1:8002"
|
||||||
|
steps = parse_runbook(runbook_path, raw, sofiia_url=base_url)
|
||||||
|
if not steps:
|
||||||
|
raise ValueError("Runbook produced no steps")
|
||||||
|
|
||||||
|
run_id = str(uuid.uuid4())
|
||||||
|
now = _now_ts()
|
||||||
|
conn = await _db.get_db()
|
||||||
|
await conn.execute(
|
||||||
|
"""INSERT INTO runbook_runs(run_id, runbook_path, status, current_step, created_at, started_at,
|
||||||
|
finished_at, operator_id, node_id, sofiia_url, data_json, evidence_path)
|
||||||
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||||
|
(
|
||||||
|
run_id,
|
||||||
|
runbook_path,
|
||||||
|
"running",
|
||||||
|
0,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
None,
|
||||||
|
operator_id or None,
|
||||||
|
node_id,
|
||||||
|
base_url,
|
||||||
|
json.dumps(data_json or {}, separators=(",", ":")) if data_json else None,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for s in steps:
|
||||||
|
await conn.execute(
|
||||||
|
"""INSERT INTO runbook_steps(run_id, step_index, title, section, action_type, action_json,
|
||||||
|
status, result_json, started_at, finished_at) VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
||||||
|
(
|
||||||
|
run_id,
|
||||||
|
s.step_index,
|
||||||
|
s.title,
|
||||||
|
s.section,
|
||||||
|
s.action_type,
|
||||||
|
json.dumps(s.action_json, separators=(",", ":")),
|
||||||
|
s.status,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await conn.commit()
|
||||||
|
|
||||||
|
await audit_log(
|
||||||
|
AuditEvent(
|
||||||
|
event="runbook.run.created",
|
||||||
|
operator_id=operator_id or "unknown",
|
||||||
|
operator_id_missing=not operator_id,
|
||||||
|
node_id=node_id,
|
||||||
|
status="ok",
|
||||||
|
data={"run_id": run_id, "runbook_path": runbook_path, "steps_total": len(steps)},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"run_id": run_id,
|
||||||
|
"status": "running",
|
||||||
|
"current_step": 0,
|
||||||
|
"steps_total": len(steps),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def get_run(run_id: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Return run row + steps (light: no large action_json/result_json)."""
|
||||||
|
conn = await _db.get_db()
|
||||||
|
async with conn.execute(
|
||||||
|
"SELECT run_id, runbook_path, status, current_step, created_at, started_at, finished_at,"
|
||||||
|
" operator_id, node_id, sofiia_url, evidence_path FROM runbook_runs WHERE run_id = ?",
|
||||||
|
(run_id,),
|
||||||
|
) as cur:
|
||||||
|
row = await cur.fetchone()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
run = {
|
||||||
|
"run_id": row[0],
|
||||||
|
"runbook_path": row[1],
|
||||||
|
"status": row[2],
|
||||||
|
"current_step": row[3],
|
||||||
|
"created_at": row[4],
|
||||||
|
"started_at": row[5],
|
||||||
|
"finished_at": row[6],
|
||||||
|
"operator_id": row[7],
|
||||||
|
"node_id": row[8],
|
||||||
|
"sofiia_url": row[9],
|
||||||
|
"evidence_path": row[10],
|
||||||
|
}
|
||||||
|
async with conn.execute(
|
||||||
|
"SELECT step_index, title, section, action_type, status, started_at, finished_at "
|
||||||
|
"FROM runbook_steps WHERE run_id = ? ORDER BY step_index",
|
||||||
|
(run_id,),
|
||||||
|
) as cur:
|
||||||
|
step_rows = await cur.fetchall()
|
||||||
|
run["steps"] = [
|
||||||
|
{
|
||||||
|
"step_index": r[0],
|
||||||
|
"title": r[1],
|
||||||
|
"section": r[2],
|
||||||
|
"action_type": r[3],
|
||||||
|
"status": r[4],
|
||||||
|
"started_at": r[5],
|
||||||
|
"finished_at": r[6],
|
||||||
|
}
|
||||||
|
for r in (step_rows or [])
|
||||||
|
]
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
async def _execute_http_check(base_url: str, action: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Allowlisted GET only; returns {status, status_code, ok, error}."""
|
||||||
|
path = (action.get("url_path") or "").strip()
|
||||||
|
if not path.startswith("/"):
|
||||||
|
path = "/" + path
|
||||||
|
if path not in _ALLOWED_HTTP_PATHS:
|
||||||
|
return {"ok": False, "error": "path not allowlisted", "status_code": None}
|
||||||
|
url = (base_url or "http://127.0.0.1:8002").rstrip("/") + path
|
||||||
|
expect_statuses = action.get("expect", {}).get("status") or [200]
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||||
|
r = await client.get(url, headers=action.get("headers") or {})
|
||||||
|
ok = r.status_code in expect_statuses
|
||||||
|
return {
|
||||||
|
"ok": ok,
|
||||||
|
"status_code": r.status_code,
|
||||||
|
"expected": expect_statuses,
|
||||||
|
"status": "ok" if ok else "fail",
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("http_check %s failed: %s", url, e)
|
||||||
|
return {"ok": False, "error": str(e)[:200], "status_code": None, "status": "fail"}
|
||||||
|
|
||||||
|
|
||||||
|
async def next_step(run_id: str, operator_id: str = "") -> Optional[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Get current step; if http_check execute and advance; if manual return instructions.
|
||||||
|
Returns either {type: "http_check", step_index, result, ...} or {type: "manual", step_index, title, instructions_md}.
|
||||||
|
"""
|
||||||
|
conn = await _db.get_db()
|
||||||
|
async with conn.execute(
|
||||||
|
"SELECT runbook_path, status, current_step, sofiia_url FROM runbook_runs WHERE run_id = ?",
|
||||||
|
(run_id,),
|
||||||
|
) as cur:
|
||||||
|
row = await cur.fetchone()
|
||||||
|
if not row or row[1] not in ("running", "paused"):
|
||||||
|
return None
|
||||||
|
runbook_path, status, current_step, sofiia_url = row[0], row[1], row[2], row[3]
|
||||||
|
|
||||||
|
async with conn.execute(
|
||||||
|
"SELECT step_index, title, section, action_type, action_json, status FROM runbook_steps "
|
||||||
|
"WHERE run_id = ? AND step_index = ?",
|
||||||
|
(run_id, current_step),
|
||||||
|
) as cur:
|
||||||
|
step_row = await cur.fetchone()
|
||||||
|
if not step_row:
|
||||||
|
return None
|
||||||
|
step_index, title, section, action_type, action_json_str, step_status = step_row
|
||||||
|
action_json = json.loads(action_json_str) if action_json_str else {}
|
||||||
|
|
||||||
|
started_at = _now_ts()
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_steps SET status = ?, started_at = ? WHERE run_id = ? AND step_index = ?",
|
||||||
|
("running", started_at, run_id, step_index),
|
||||||
|
)
|
||||||
|
await conn.commit()
|
||||||
|
await audit_log(
|
||||||
|
AuditEvent(
|
||||||
|
event="runbook.step.started",
|
||||||
|
operator_id=operator_id or "unknown",
|
||||||
|
node_id=None,
|
||||||
|
status="ok",
|
||||||
|
data={"run_id": run_id, "step_index": step_index, "action_type": action_type, "title": title},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if action_type == "http_check":
|
||||||
|
result = await _execute_http_check(sofiia_url or "http://127.0.0.1:8002", action_json)
|
||||||
|
finished_at = _now_ts()
|
||||||
|
duration_ms = int((finished_at - started_at) * 1000)
|
||||||
|
step_status = "ok" if result.get("ok") else "fail"
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_steps SET status = ?, result_json = ?, finished_at = ? WHERE run_id = ? AND step_index = ?",
|
||||||
|
(step_status, json.dumps(result, separators=(",", ":")), finished_at, run_id, step_index),
|
||||||
|
)
|
||||||
|
next_current = current_step + 1
|
||||||
|
async with conn.execute("SELECT COUNT(*) FROM runbook_steps WHERE run_id = ?", (run_id,)) as cur:
|
||||||
|
total = (await cur.fetchone())[0]
|
||||||
|
if next_current >= total:
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_runs SET current_step = ?, status = ?, finished_at = ? WHERE run_id = ?",
|
||||||
|
(next_current, "completed", finished_at, run_id),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await conn.execute("UPDATE runbook_runs SET current_step = ? WHERE run_id = ?", (next_current, run_id))
|
||||||
|
await conn.commit()
|
||||||
|
await audit_log(
|
||||||
|
AuditEvent(
|
||||||
|
event="runbook.step.completed",
|
||||||
|
operator_id=operator_id or "unknown",
|
||||||
|
node_id=None,
|
||||||
|
status=step_status,
|
||||||
|
duration_ms=duration_ms,
|
||||||
|
data={
|
||||||
|
"run_id": run_id,
|
||||||
|
"step_index": step_index,
|
||||||
|
"action_type": action_type,
|
||||||
|
"result_ok": result.get("ok"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"type": "http_check",
|
||||||
|
"step_index": step_index,
|
||||||
|
"title": title,
|
||||||
|
"result": result,
|
||||||
|
"next_step": next_current,
|
||||||
|
"completed": next_current >= total,
|
||||||
|
}
|
||||||
|
|
||||||
|
# manual
|
||||||
|
instructions = action_json.get("instructions_md") or title
|
||||||
|
return {
|
||||||
|
"type": "manual",
|
||||||
|
"step_index": step_index,
|
||||||
|
"title": title,
|
||||||
|
"section": section,
|
||||||
|
"instructions_md": instructions,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def complete_step(
|
||||||
|
run_id: str,
|
||||||
|
step_index: int,
|
||||||
|
status: str = "ok",
|
||||||
|
notes: str = "",
|
||||||
|
data: Optional[Dict[str, Any]] = None,
|
||||||
|
operator_id: str = "",
|
||||||
|
) -> bool:
|
||||||
|
"""Record manual step completion, advance current_step."""
|
||||||
|
if status not in ("ok", "warn", "fail", "skipped"):
|
||||||
|
status = "ok"
|
||||||
|
conn = await _db.get_db()
|
||||||
|
async with conn.execute(
|
||||||
|
"SELECT current_step FROM runbook_runs WHERE run_id = ? AND runbook_runs.status IN ('running','paused')",
|
||||||
|
(run_id,),
|
||||||
|
) as cur:
|
||||||
|
row = await cur.fetchone()
|
||||||
|
if not row or row[0] != step_index:
|
||||||
|
return False
|
||||||
|
finished_at = _now_ts()
|
||||||
|
result_json = json.dumps({"status": status, "notes": notes[:500], **(data or {})}, separators=(",", ":"))
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_steps SET status = ?, result_json = ?, finished_at = ? WHERE run_id = ? AND step_index = ?",
|
||||||
|
(status, result_json, finished_at, run_id, step_index),
|
||||||
|
)
|
||||||
|
next_current = step_index + 1
|
||||||
|
async with conn.execute("SELECT COUNT(*) FROM runbook_steps WHERE run_id = ?", (run_id,)) as cur:
|
||||||
|
total = (await cur.fetchone())[0]
|
||||||
|
if next_current >= total:
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_runs SET current_step = ?, status = ?, finished_at = ? WHERE run_id = ?",
|
||||||
|
(next_current, "completed", finished_at, run_id),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await conn.execute("UPDATE runbook_runs SET current_step = ? WHERE run_id = ?", (next_current, run_id))
|
||||||
|
await conn.commit()
|
||||||
|
await audit_log(
|
||||||
|
AuditEvent(
|
||||||
|
event="runbook.step.completed",
|
||||||
|
operator_id=operator_id or "unknown",
|
||||||
|
status=status,
|
||||||
|
data={"run_id": run_id, "step_index": step_index, "action_type": "manual"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def abort_run(run_id: str, operator_id: str = "") -> bool:
|
||||||
|
"""Set status=aborted, audit."""
|
||||||
|
conn = await _db.get_db()
|
||||||
|
async with conn.execute("SELECT 1 FROM runbook_runs WHERE run_id = ?", (run_id,)) as cur:
|
||||||
|
if not await cur.fetchone():
|
||||||
|
return False
|
||||||
|
now = _now_ts()
|
||||||
|
await conn.execute(
|
||||||
|
"UPDATE runbook_runs SET status = ?, finished_at = ? WHERE run_id = ?",
|
||||||
|
("aborted", now, run_id),
|
||||||
|
)
|
||||||
|
await conn.commit()
|
||||||
|
await audit_log(
|
||||||
|
AuditEvent(
|
||||||
|
event="runbook.run.aborted",
|
||||||
|
operator_id=operator_id or "unknown",
|
||||||
|
status="ok",
|
||||||
|
data={"run_id": run_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return True
|
||||||
94
services/sofiia-console/app/runbook_runs_router.py
Normal file
94
services/sofiia-console/app/runbook_runs_router.py
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
"""
|
||||||
|
Runbook runs API — create run, get run, next step, complete step, abort (PR2).
|
||||||
|
All under require_auth.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Body
|
||||||
|
|
||||||
|
from .auth import require_auth
|
||||||
|
from . import runbook_runner as runner
|
||||||
|
|
||||||
|
runbook_runs_router = APIRouter(prefix="/api/runbooks/runs", tags=["runbook-runs"])
|
||||||
|
|
||||||
|
|
||||||
|
@runbook_runs_router.post("")
|
||||||
|
async def create_run(
|
||||||
|
body: Dict[str, Any] = Body(...),
|
||||||
|
_auth: str = Depends(require_auth),
|
||||||
|
):
|
||||||
|
"""Create a new run from runbook_path. Returns run_id, status, current_step, steps_total."""
|
||||||
|
runbook_path = (body.get("runbook_path") or "").strip()
|
||||||
|
if not runbook_path or ".." in runbook_path:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid runbook_path")
|
||||||
|
try:
|
||||||
|
out = await runner.create_run(
|
||||||
|
runbook_path=runbook_path,
|
||||||
|
operator_id=(body.get("operator_id") or "").strip() or None,
|
||||||
|
node_id=(body.get("node_id") or "").strip() or None,
|
||||||
|
sofiia_url=(body.get("sofiia_url") or "").strip() or None,
|
||||||
|
data_json=body.get("data"),
|
||||||
|
)
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@runbook_runs_router.get("/{run_id}")
|
||||||
|
async def get_run(
|
||||||
|
run_id: str,
|
||||||
|
_auth: str = Depends(require_auth),
|
||||||
|
):
|
||||||
|
"""Get run and steps (light)."""
|
||||||
|
run = await runner.get_run(run_id)
|
||||||
|
if not run:
|
||||||
|
raise HTTPException(status_code=404, detail="Run not found")
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
@runbook_runs_router.post("/{run_id}/next")
|
||||||
|
async def next_step(
|
||||||
|
run_id: str,
|
||||||
|
body: Dict[str, Any] = Body(default=None),
|
||||||
|
_auth: str = Depends(require_auth),
|
||||||
|
):
|
||||||
|
"""Execute next step: if http_check run it and return result; if manual return instructions."""
|
||||||
|
operator_id = (body or {}).get("operator_id") or ""
|
||||||
|
out = await runner.next_step(run_id, operator_id=operator_id)
|
||||||
|
if out is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Run not found or not active")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@runbook_runs_router.post("/{run_id}/steps/{step_index}/complete")
|
||||||
|
async def complete_step(
|
||||||
|
run_id: str,
|
||||||
|
step_index: int,
|
||||||
|
body: Dict[str, Any] = Body(...),
|
||||||
|
_auth: str = Depends(require_auth),
|
||||||
|
):
|
||||||
|
"""Mark manual step complete. Body: status (ok|warn|fail|skipped), notes, data."""
|
||||||
|
status = (body.get("status") or "ok").strip()
|
||||||
|
notes = (body.get("notes") or "").strip()
|
||||||
|
data = body.get("data")
|
||||||
|
operator_id = (body.get("operator_id") or "").strip()
|
||||||
|
ok = await runner.complete_step(run_id, step_index, status=status, notes=notes, data=data, operator_id=operator_id)
|
||||||
|
if not ok:
|
||||||
|
raise HTTPException(status_code=404, detail="Run/step not found or step not current")
|
||||||
|
return {"ok": True, "run_id": run_id, "step_index": step_index, "next_step": step_index + 1}
|
||||||
|
|
||||||
|
|
||||||
|
@runbook_runs_router.post("/{run_id}/abort")
|
||||||
|
async def abort_run(
|
||||||
|
run_id: str,
|
||||||
|
body: Dict[str, Any] = Body(default=None),
|
||||||
|
_auth: str = Depends(require_auth),
|
||||||
|
):
|
||||||
|
"""Abort run."""
|
||||||
|
operator_id = (body or {}).get("operator_id") or ""
|
||||||
|
ok = await runner.abort_run(run_id, operator_id=operator_id)
|
||||||
|
if not ok:
|
||||||
|
raise HTTPException(status_code=404, detail="Run not found")
|
||||||
|
return {"ok": True, "run_id": run_id, "status": "aborted"}
|
||||||
176
tests/test_sofiia_runbook_runner.py
Normal file
176
tests/test_sofiia_runbook_runner.py
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
"""
|
||||||
|
Tests for runbook runner (PR2): create run, next (http_check + manual), complete, abort.
|
||||||
|
Uses tmp docs + rebuilt index; http_check monkeypatched to avoid network.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def tmp_docs_rehearsal(tmp_path):
|
||||||
|
"""Create tmp_path/docs/runbook/rehearsal-v1-30min-checklist.md."""
|
||||||
|
docs_root = tmp_path / "docs"
|
||||||
|
runbook_dir = docs_root / "runbook"
|
||||||
|
runbook_dir.mkdir(parents=True)
|
||||||
|
(runbook_dir / "rehearsal-v1-30min-checklist.md").write_text(
|
||||||
|
"# Rehearsal v1 — 30-minute execution plan\n\n"
|
||||||
|
"## 0–5 min — Preflight\n\n"
|
||||||
|
"Run STRICT=1 bash ops/preflight_sofiia_console.sh\n\n"
|
||||||
|
"## 5–10 min — Restart\n\n"
|
||||||
|
"Restart sofiia-console.",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return docs_root
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_run_returns_run_id_and_steps_total(sofiia_module, tmp_path, tmp_docs_rehearsal, monkeypatch):
|
||||||
|
"""Create run with rehearsal runbook returns run_id, status, steps_total >= 1."""
|
||||||
|
import app.docs_index as docs_index_mod
|
||||||
|
import app.runbook_runner as runner_mod
|
||||||
|
|
||||||
|
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
async def run():
|
||||||
|
await docs_index_mod.rebuild_index(tmp_docs_rehearsal)
|
||||||
|
return await runner_mod.create_run(
|
||||||
|
"runbook/rehearsal-v1-30min-checklist.md",
|
||||||
|
operator_id="test-op",
|
||||||
|
sofiia_url="http://127.0.0.1:8002",
|
||||||
|
)
|
||||||
|
|
||||||
|
out = loop.run_until_complete(run())
|
||||||
|
assert "run_id" in out
|
||||||
|
assert out["status"] == "running"
|
||||||
|
assert out["current_step"] == 0
|
||||||
|
assert out["steps_total"] >= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_run_404_for_unknown_run_id(sofiia_client):
|
||||||
|
"""GET /api/runbooks/runs/{run_id} returns 404 for unknown run_id."""
|
||||||
|
r = sofiia_client.get("/api/runbooks/runs/00000000-0000-0000-0000-000000000000")
|
||||||
|
assert r.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_run_400_for_invalid_path(sofiia_client):
|
||||||
|
"""POST /api/runbooks/runs with invalid runbook_path returns 400."""
|
||||||
|
r = sofiia_client.post(
|
||||||
|
"/api/runbooks/runs",
|
||||||
|
json={"runbook_path": "../../../etc/passwd"},
|
||||||
|
)
|
||||||
|
assert r.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_next_step_http_check_then_manual(sofiia_module, sofiia_client, tmp_path, tmp_docs_rehearsal, monkeypatch):
|
||||||
|
"""Create run, call next 3 times (http_check mocked 200), then next returns manual step."""
|
||||||
|
import app.docs_index as docs_index_mod
|
||||||
|
import app.runbook_runner as runner_mod
|
||||||
|
|
||||||
|
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
async def create():
|
||||||
|
await docs_index_mod.rebuild_index(tmp_docs_rehearsal)
|
||||||
|
return await runner_mod.create_run(
|
||||||
|
"runbook/rehearsal-v1-30min-checklist.md",
|
||||||
|
sofiia_url="http://127.0.0.1:8002",
|
||||||
|
)
|
||||||
|
|
||||||
|
create_out = loop.run_until_complete(create())
|
||||||
|
run_id = create_out["run_id"]
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get:
|
||||||
|
def resp_for_url(url):
|
||||||
|
return type("Res", (), {"status_code": 401 if "audit" in str(url) else 200})()
|
||||||
|
|
||||||
|
async def fake_get(url, **kwargs):
|
||||||
|
return resp_for_url(url)
|
||||||
|
|
||||||
|
mock_get.side_effect = fake_get
|
||||||
|
for _ in range(3):
|
||||||
|
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/next", json={})
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
data = r.json()
|
||||||
|
assert data.get("type") in ("http_check", "manual")
|
||||||
|
if data.get("type") == "http_check":
|
||||||
|
assert data.get("result", {}).get("ok") is True
|
||||||
|
|
||||||
|
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/next", json={})
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
data = r.json()
|
||||||
|
assert data.get("type") == "manual"
|
||||||
|
assert "instructions_md" in data or "title" in data
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_step_advances_run(sofiia_module, sofiia_client, tmp_path, tmp_docs_rehearsal, monkeypatch):
|
||||||
|
"""After manual step returned by next, complete_step advances and returns ok."""
|
||||||
|
import app.docs_index as docs_index_mod
|
||||||
|
import app.runbook_runner as runner_mod
|
||||||
|
|
||||||
|
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
async def create():
|
||||||
|
await docs_index_mod.rebuild_index(tmp_docs_rehearsal)
|
||||||
|
return await runner_mod.create_run(
|
||||||
|
"runbook/rehearsal-v1-30min-checklist.md",
|
||||||
|
sofiia_url="http://127.0.0.1:8002",
|
||||||
|
)
|
||||||
|
|
||||||
|
create_out = loop.run_until_complete(create())
|
||||||
|
run_id = create_out["run_id"]
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get:
|
||||||
|
def resp(url):
|
||||||
|
return type("Res", (), {"status_code": 401 if "audit" in str(url) else 200})()
|
||||||
|
|
||||||
|
async def fake_get(url, **kwargs):
|
||||||
|
return resp(url)
|
||||||
|
|
||||||
|
mock_get.side_effect = fake_get
|
||||||
|
for _ in range(3):
|
||||||
|
sofiia_client.post(f"/api/runbooks/runs/{run_id}/next", json={})
|
||||||
|
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/next", json={})
|
||||||
|
assert r.status_code == 200
|
||||||
|
manual = r.json()
|
||||||
|
assert manual.get("type") == "manual"
|
||||||
|
step_index = manual["step_index"]
|
||||||
|
|
||||||
|
r2 = sofiia_client.post(
|
||||||
|
f"/api/runbooks/runs/{run_id}/steps/{step_index}/complete",
|
||||||
|
json={"status": "ok", "notes": "done"},
|
||||||
|
)
|
||||||
|
assert r2.status_code == 200, r2.text
|
||||||
|
assert r2.json().get("ok") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_abort_run(sofiia_module, sofiia_client, tmp_path, tmp_docs_rehearsal, monkeypatch):
|
||||||
|
"""Create run, abort returns ok; get_run then shows status aborted."""
|
||||||
|
import app.docs_index as docs_index_mod
|
||||||
|
import app.runbook_runner as runner_mod
|
||||||
|
|
||||||
|
monkeypatch.setenv("SOFIIA_DATA_DIR", str(tmp_path / "sofiia-data"))
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
async def create():
|
||||||
|
await docs_index_mod.rebuild_index(tmp_docs_rehearsal)
|
||||||
|
return await runner_mod.create_run(
|
||||||
|
"runbook/rehearsal-v1-30min-checklist.md",
|
||||||
|
sofiia_url="http://127.0.0.1:8002",
|
||||||
|
)
|
||||||
|
|
||||||
|
create_out = loop.run_until_complete(create())
|
||||||
|
run_id = create_out["run_id"]
|
||||||
|
|
||||||
|
r = sofiia_client.post(f"/api/runbooks/runs/{run_id}/abort", json={})
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
assert r.json().get("status") == "aborted"
|
||||||
|
|
||||||
|
run = loop.run_until_complete(runner_mod.get_run(run_id))
|
||||||
|
assert run is not None
|
||||||
|
assert run["status"] == "aborted"
|
||||||
Reference in New Issue
Block a user