diff --git a/ops/canary_all.sh b/ops/canary_all.sh index 7141ea8d..868d5ac5 100755 --- a/ops/canary_all.sh +++ b/ops/canary_all.sh @@ -13,5 +13,6 @@ run() { run "gateway_delivery_priority" "$ROOT/ops/canary_gateway_delivery_priority.sh" run "router_contract" "$ROOT/ops/canary_router_contract.sh" +run "daarwizz_awareness" "$ROOT/ops/check_daarwizz_awareness.sh" echo "[OK] all canaries passed" diff --git a/ops/check_daarwizz_awareness.sh b/ops/check_daarwizz_awareness.sh new file mode 100755 index 00000000..c5f59f93 --- /dev/null +++ b/ops/check_daarwizz_awareness.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROUTER_URL="${ROUTER_URL:-http://127.0.0.1:9102}" +REQUEST_TIMEOUT_SEC="${DAARWIZZ_AWARENESS_TIMEOUT_SEC:-35}" +OUT_JSON="${DAARWIZZ_AWARENESS_OUT_JSON:-/tmp/daarwizz_awareness_audit.json}" + +PYTHON_BIN="python3" +if [[ -x "$ROOT/.venv-codex/bin/python" ]]; then + PYTHON_BIN="$ROOT/.venv-codex/bin/python" +fi + +echo "[INFO] checking DAARWIZZ awareness for top-level agents" +echo "[INFO] router=$ROUTER_URL timeout=${REQUEST_TIMEOUT_SEC}s" + +"$PYTHON_BIN" - "$ROOT" "$ROUTER_URL" "$REQUEST_TIMEOUT_SEC" "$OUT_JSON" <<'PY' +import json +import re +import sys +import time +from pathlib import Path +from urllib import request as urlreq + +import yaml + +root = Path(sys.argv[1]) +router = sys.argv[2].rstrip("/") +timeout_s = int(sys.argv[3]) +out_json = Path(sys.argv[4]) + +registry = yaml.safe_load((root / "config" / "agent_registry.yml").read_text(encoding="utf-8")) +agents = [a for a in registry.get("agents", []) if a.get("class") == "top_level"] + +prompt = ( + "Who is DAARWIZZ in DAARION network? " + "Answer briefly in 1 sentence and mention that DAARWIZZ is the main orchestrator/mayor." +) + +deny_patterns = [ + r"не знаю\s+daarwizz", + r"don't know\s+daarwizz", + r"do not know\s+daarwizz", + r"no information about daarwizz", + r"не\s+знаю\s+хто\s+це", +] +pos_patterns = [ + r"оркестратор", + r"orchestrator", + r"мер", + r"mayor", + r"entry point", + r"вхідн", +] + +rows = [] +for idx, agent in enumerate(agents, 1): + agent_id = agent["id"] + req_body = {"prompt": prompt, "max_tokens": 100, "temperature": 0.1} + req = urlreq.Request( + url=f"{router}/v1/agents/{agent_id}/infer", + data=json.dumps(req_body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + row = {"agent": agent_id, "status": "FAIL", "reason": "", "preview": ""} + try: + with urlreq.urlopen(req, timeout=timeout_s) as resp: + data = json.loads(resp.read().decode("utf-8", errors="ignore")) + text = (data.get("response") or "").strip() + low = text.lower() + row["preview"] = text[:220].replace("\n", " ") + + deny = any(re.search(p, low) for p in deny_patterns) + pos = any(re.search(p, low) for p in pos_patterns) + if deny: + row["status"] = "FAIL" + row["reason"] = "explicit_deny" + elif pos: + row["status"] = "PASS" + row["reason"] = "role_context_found" + else: + row["status"] = "FAIL" + row["reason"] = "no_role_context" + except Exception as e: + row["status"] = "FAIL" + row["reason"] = f"http_error:{type(e).__name__}" + + rows.append(row) + print(f"[{idx:02d}/{len(agents)}] {agent_id}: {row['status']} ({row['reason']})") + time.sleep(0.05) + +summary = { + "total": len(rows), + "pass": sum(1 for r in rows if r["status"] == "PASS"), + "fail": sum(1 for r in rows if r["status"] == "FAIL"), +} + +payload = {"summary": summary, "results": rows} +out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +print(json.dumps(summary, ensure_ascii=False)) +print(f"[INFO] details: {out_json}") + +if summary["fail"] > 0: + failed = [r["agent"] for r in rows if r["status"] == "FAIL"] + raise SystemExit(f"[FAIL] DAARWIZZ awareness failed for: {', '.join(failed)}") + +print("[OK] DAARWIZZ awareness canary passed") +PY