From e5a6e310b7de9df07cce8341fefc207a5e7ae169 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 18 Feb 2026 08:29:02 -0800 Subject: [PATCH] ops: make DAARWIZZ awareness canary static by default with optional runtime mode --- ops/check_daarwizz_awareness.sh | 173 +++++++++++++++++++------------- 1 file changed, 106 insertions(+), 67 deletions(-) diff --git a/ops/check_daarwizz_awareness.sh b/ops/check_daarwizz_awareness.sh index c5f59f93..11085e86 100755 --- a/ops/check_daarwizz_awareness.sh +++ b/ops/check_daarwizz_awareness.sh @@ -2,6 +2,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODE="${DAARWIZZ_AWARENESS_MODE:-static}" # static | runtime ROUTER_URL="${ROUTER_URL:-http://127.0.0.1:9102}" REQUEST_TIMEOUT_SEC="${DAARWIZZ_AWARENESS_TIMEOUT_SEC:-35}" OUT_JSON="${DAARWIZZ_AWARENESS_OUT_JSON:-/tmp/daarwizz_awareness_audit.json}" @@ -11,10 +12,9 @@ if [[ -x "$ROOT/.venv-codex/bin/python" ]]; then PYTHON_BIN="$ROOT/.venv-codex/bin/python" fi -echo "[INFO] checking DAARWIZZ awareness for top-level agents" -echo "[INFO] router=$ROUTER_URL timeout=${REQUEST_TIMEOUT_SEC}s" +echo "[INFO] checking DAARWIZZ awareness (mode=$MODE)" -"$PYTHON_BIN" - "$ROOT" "$ROUTER_URL" "$REQUEST_TIMEOUT_SEC" "$OUT_JSON" <<'PY' +"$PYTHON_BIN" - "$ROOT" "$MODE" "$ROUTER_URL" "$REQUEST_TIMEOUT_SEC" "$OUT_JSON" <<'PY' import json import re import sys @@ -25,86 +25,125 @@ from urllib import request as urlreq import yaml root = Path(sys.argv[1]) -router = sys.argv[2].rstrip("/") -timeout_s = int(sys.argv[3]) -out_json = Path(sys.argv[4]) +mode = sys.argv[2] +router = sys.argv[3].rstrip("/") +timeout_s = int(sys.argv[4]) +out_json = Path(sys.argv[5]) registry = yaml.safe_load((root / "config" / "agent_registry.yml").read_text(encoding="utf-8")) agents = [a for a in registry.get("agents", []) if a.get("class") == "top_level"] -prompt = ( - "Who is DAARWIZZ in DAARION network? " - "Answer briefly in 1 sentence and mention that DAARWIZZ is the main orchestrator/mayor." -) +results = [] -deny_patterns = [ - r"не знаю\s+daarwizz", - r"don't know\s+daarwizz", - r"do not know\s+daarwizz", - r"no information about daarwizz", - r"не\s+знаю\s+хто\s+це", -] -pos_patterns = [ - r"оркестратор", - r"orchestrator", - r"мер", - r"mayor", - r"entry point", - r"вхідн", -] +if mode == "static": + for idx, agent in enumerate(agents, 1): + aid = agent["id"] + prompt_file = agent.get("prompt_file", "") + row = {"agent": aid, "status": "FAIL", "reason": "", "preview": ""} -rows = [] -for idx, agent in enumerate(agents, 1): - agent_id = agent["id"] - req_body = {"prompt": prompt, "max_tokens": 100, "temperature": 0.1} - req = urlreq.Request( - url=f"{router}/v1/agents/{agent_id}/infer", - data=json.dumps(req_body).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", + # DAARWIZZ itself can be defined without self-reference check. + if aid == "daarwizz": + row["status"] = "PASS" + row["reason"] = "self_agent" + results.append(row) + print(f"[{idx:02d}/{len(agents)}] {aid}: PASS (self_agent)", flush=True) + continue + + if not prompt_file: + row["reason"] = "missing_prompt_file" + results.append(row) + print(f"[{idx:02d}/{len(agents)}] {aid}: FAIL (missing_prompt_file)", flush=True) + continue + + prompt_path = root / "gateway-bot" / prompt_file + if not prompt_path.exists(): + row["reason"] = f"prompt_not_found:{prompt_file}" + results.append(row) + print(f"[{idx:02d}/{len(agents)}] {aid}: FAIL ({row['reason']})", flush=True) + continue + + text = prompt_path.read_text(encoding="utf-8", errors="ignore") + low = text.lower() + row["preview"] = text[:180].replace("\n", " ") + + if "daarwizz" in low: + row["status"] = "PASS" + row["reason"] = "prompt_mentions_daarwizz" + else: + row["status"] = "FAIL" + row["reason"] = "prompt_missing_daarwizz_reference" + + results.append(row) + print(f"[{idx:02d}/{len(agents)}] {aid}: {row['status']} ({row['reason']})", flush=True) + +elif mode == "runtime": + prompt = ( + "Who is DAARWIZZ in DAARION network? " + "Answer briefly in 1 sentence and mention that DAARWIZZ is the main orchestrator/mayor." ) + deny_patterns = [ + r"не знаю\s+daarwizz", + r"don't know\s+daarwizz", + r"do not know\s+daarwizz", + r"no information about daarwizz", + r"не\s+знаю\s+хто\s+це", + ] + pos_patterns = [r"оркестратор", r"orchestrator", r"мер", r"mayor", r"entry point", r"вхідн"] - row = {"agent": agent_id, "status": "FAIL", "reason": "", "preview": ""} - try: - with urlreq.urlopen(req, timeout=timeout_s) as resp: - data = json.loads(resp.read().decode("utf-8", errors="ignore")) - text = (data.get("response") or "").strip() - low = text.lower() - row["preview"] = text[:220].replace("\n", " ") + for idx, agent in enumerate(agents, 1): + aid = agent["id"] + req_body = {"prompt": prompt, "max_tokens": 100, "temperature": 0.1} + req = urlreq.Request( + url=f"{router}/v1/agents/{aid}/infer", + data=json.dumps(req_body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) - deny = any(re.search(p, low) for p in deny_patterns) - pos = any(re.search(p, low) for p in pos_patterns) - if deny: - row["status"] = "FAIL" - row["reason"] = "explicit_deny" - elif pos: - row["status"] = "PASS" - row["reason"] = "role_context_found" - else: - row["status"] = "FAIL" - row["reason"] = "no_role_context" - except Exception as e: - row["status"] = "FAIL" - row["reason"] = f"http_error:{type(e).__name__}" + row = {"agent": aid, "status": "FAIL", "reason": "", "preview": ""} + try: + with urlreq.urlopen(req, timeout=timeout_s) as resp: + data = json.loads(resp.read().decode("utf-8", errors="ignore")) + text = (data.get("response") or "").strip() + low = text.lower() + row["preview"] = text[:220].replace("\n", " ") - rows.append(row) - print(f"[{idx:02d}/{len(agents)}] {agent_id}: {row['status']} ({row['reason']})") - time.sleep(0.05) + deny = any(re.search(p, low) for p in deny_patterns) + pos = any(re.search(p, low) for p in pos_patterns) + if deny: + row["status"] = "FAIL" + row["reason"] = "explicit_deny" + elif pos: + row["status"] = "PASS" + row["reason"] = "role_context_found" + else: + row["status"] = "FAIL" + row["reason"] = "no_role_context" + except Exception as e: + row["status"] = "FAIL" + row["reason"] = f"http_error:{type(e).__name__}" + + results.append(row) + print(f"[{idx:02d}/{len(agents)}] {aid}: {row['status']} ({row['reason']})", flush=True) + time.sleep(0.05) +else: + raise SystemExit(f"[FAIL] unsupported mode: {mode}") summary = { - "total": len(rows), - "pass": sum(1 for r in rows if r["status"] == "PASS"), - "fail": sum(1 for r in rows if r["status"] == "FAIL"), + "mode": mode, + "total": len(results), + "pass": sum(1 for r in results if r["status"] == "PASS"), + "fail": sum(1 for r in results if r["status"] == "FAIL"), } - -payload = {"summary": summary, "results": rows} +payload = {"summary": summary, "results": results} out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") -print(json.dumps(summary, ensure_ascii=False)) -print(f"[INFO] details: {out_json}") + +print(json.dumps(summary, ensure_ascii=False), flush=True) +print(f"[INFO] details: {out_json}", flush=True) if summary["fail"] > 0: - failed = [r["agent"] for r in rows if r["status"] == "FAIL"] + failed = [r["agent"] for r in results if r["status"] == "FAIL"] raise SystemExit(f"[FAIL] DAARWIZZ awareness failed for: {', '.join(failed)}") -print("[OK] DAARWIZZ awareness canary passed") +print("[OK] DAARWIZZ awareness canary passed", flush=True) PY