ops: make DAARWIZZ awareness canary static by default with optional runtime mode
This commit is contained in:
Apple
@@ -2,6 +2,7 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
MODE="${DAARWIZZ_AWARENESS_MODE:-static}" # static | runtime
|
||||||
ROUTER_URL="${ROUTER_URL:-http://127.0.0.1:9102}"
|
ROUTER_URL="${ROUTER_URL:-http://127.0.0.1:9102}"
|
||||||
REQUEST_TIMEOUT_SEC="${DAARWIZZ_AWARENESS_TIMEOUT_SEC:-35}"
|
REQUEST_TIMEOUT_SEC="${DAARWIZZ_AWARENESS_TIMEOUT_SEC:-35}"
|
||||||
OUT_JSON="${DAARWIZZ_AWARENESS_OUT_JSON:-/tmp/daarwizz_awareness_audit.json}"
|
OUT_JSON="${DAARWIZZ_AWARENESS_OUT_JSON:-/tmp/daarwizz_awareness_audit.json}"
|
||||||
@@ -11,10 +12,9 @@ if [[ -x "$ROOT/.venv-codex/bin/python" ]]; then
|
|||||||
PYTHON_BIN="$ROOT/.venv-codex/bin/python"
|
PYTHON_BIN="$ROOT/.venv-codex/bin/python"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "[INFO] checking DAARWIZZ awareness for top-level agents"
|
echo "[INFO] checking DAARWIZZ awareness (mode=$MODE)"
|
||||||
echo "[INFO] router=$ROUTER_URL timeout=${REQUEST_TIMEOUT_SEC}s"
|
|
||||||
|
|
||||||
"$PYTHON_BIN" - "$ROOT" "$ROUTER_URL" "$REQUEST_TIMEOUT_SEC" "$OUT_JSON" <<'PY'
|
"$PYTHON_BIN" - "$ROOT" "$MODE" "$ROUTER_URL" "$REQUEST_TIMEOUT_SEC" "$OUT_JSON" <<'PY'
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@@ -25,86 +25,125 @@ from urllib import request as urlreq
|
|||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
root = Path(sys.argv[1])
|
root = Path(sys.argv[1])
|
||||||
router = sys.argv[2].rstrip("/")
|
mode = sys.argv[2]
|
||||||
timeout_s = int(sys.argv[3])
|
router = sys.argv[3].rstrip("/")
|
||||||
out_json = Path(sys.argv[4])
|
timeout_s = int(sys.argv[4])
|
||||||
|
out_json = Path(sys.argv[5])
|
||||||
|
|
||||||
registry = yaml.safe_load((root / "config" / "agent_registry.yml").read_text(encoding="utf-8"))
|
registry = yaml.safe_load((root / "config" / "agent_registry.yml").read_text(encoding="utf-8"))
|
||||||
agents = [a for a in registry.get("agents", []) if a.get("class") == "top_level"]
|
agents = [a for a in registry.get("agents", []) if a.get("class") == "top_level"]
|
||||||
|
|
||||||
prompt = (
|
results = []
|
||||||
"Who is DAARWIZZ in DAARION network? "
|
|
||||||
"Answer briefly in 1 sentence and mention that DAARWIZZ is the main orchestrator/mayor."
|
|
||||||
)
|
|
||||||
|
|
||||||
deny_patterns = [
|
if mode == "static":
|
||||||
r"не знаю\s+daarwizz",
|
for idx, agent in enumerate(agents, 1):
|
||||||
r"don't know\s+daarwizz",
|
aid = agent["id"]
|
||||||
r"do not know\s+daarwizz",
|
prompt_file = agent.get("prompt_file", "")
|
||||||
r"no information about daarwizz",
|
row = {"agent": aid, "status": "FAIL", "reason": "", "preview": ""}
|
||||||
r"не\s+знаю\s+хто\s+це",
|
|
||||||
]
|
|
||||||
pos_patterns = [
|
|
||||||
r"оркестратор",
|
|
||||||
r"orchestrator",
|
|
||||||
r"мер",
|
|
||||||
r"mayor",
|
|
||||||
r"entry point",
|
|
||||||
r"вхідн",
|
|
||||||
]
|
|
||||||
|
|
||||||
rows = []
|
# DAARWIZZ itself can be defined without self-reference check.
|
||||||
for idx, agent in enumerate(agents, 1):
|
if aid == "daarwizz":
|
||||||
agent_id = agent["id"]
|
row["status"] = "PASS"
|
||||||
req_body = {"prompt": prompt, "max_tokens": 100, "temperature": 0.1}
|
row["reason"] = "self_agent"
|
||||||
req = urlreq.Request(
|
results.append(row)
|
||||||
url=f"{router}/v1/agents/{agent_id}/infer",
|
print(f"[{idx:02d}/{len(agents)}] {aid}: PASS (self_agent)", flush=True)
|
||||||
data=json.dumps(req_body).encode("utf-8"),
|
continue
|
||||||
headers={"Content-Type": "application/json"},
|
|
||||||
method="POST",
|
if not prompt_file:
|
||||||
|
row["reason"] = "missing_prompt_file"
|
||||||
|
results.append(row)
|
||||||
|
print(f"[{idx:02d}/{len(agents)}] {aid}: FAIL (missing_prompt_file)", flush=True)
|
||||||
|
continue
|
||||||
|
|
||||||
|
prompt_path = root / "gateway-bot" / prompt_file
|
||||||
|
if not prompt_path.exists():
|
||||||
|
row["reason"] = f"prompt_not_found:{prompt_file}"
|
||||||
|
results.append(row)
|
||||||
|
print(f"[{idx:02d}/{len(agents)}] {aid}: FAIL ({row['reason']})", flush=True)
|
||||||
|
continue
|
||||||
|
|
||||||
|
text = prompt_path.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
low = text.lower()
|
||||||
|
row["preview"] = text[:180].replace("\n", " ")
|
||||||
|
|
||||||
|
if "daarwizz" in low:
|
||||||
|
row["status"] = "PASS"
|
||||||
|
row["reason"] = "prompt_mentions_daarwizz"
|
||||||
|
else:
|
||||||
|
row["status"] = "FAIL"
|
||||||
|
row["reason"] = "prompt_missing_daarwizz_reference"
|
||||||
|
|
||||||
|
results.append(row)
|
||||||
|
print(f"[{idx:02d}/{len(agents)}] {aid}: {row['status']} ({row['reason']})", flush=True)
|
||||||
|
|
||||||
|
elif mode == "runtime":
|
||||||
|
prompt = (
|
||||||
|
"Who is DAARWIZZ in DAARION network? "
|
||||||
|
"Answer briefly in 1 sentence and mention that DAARWIZZ is the main orchestrator/mayor."
|
||||||
)
|
)
|
||||||
|
deny_patterns = [
|
||||||
|
r"не знаю\s+daarwizz",
|
||||||
|
r"don't know\s+daarwizz",
|
||||||
|
r"do not know\s+daarwizz",
|
||||||
|
r"no information about daarwizz",
|
||||||
|
r"не\s+знаю\s+хто\s+це",
|
||||||
|
]
|
||||||
|
pos_patterns = [r"оркестратор", r"orchestrator", r"мер", r"mayor", r"entry point", r"вхідн"]
|
||||||
|
|
||||||
row = {"agent": agent_id, "status": "FAIL", "reason": "", "preview": ""}
|
for idx, agent in enumerate(agents, 1):
|
||||||
try:
|
aid = agent["id"]
|
||||||
with urlreq.urlopen(req, timeout=timeout_s) as resp:
|
req_body = {"prompt": prompt, "max_tokens": 100, "temperature": 0.1}
|
||||||
data = json.loads(resp.read().decode("utf-8", errors="ignore"))
|
req = urlreq.Request(
|
||||||
text = (data.get("response") or "").strip()
|
url=f"{router}/v1/agents/{aid}/infer",
|
||||||
low = text.lower()
|
data=json.dumps(req_body).encode("utf-8"),
|
||||||
row["preview"] = text[:220].replace("\n", " ")
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
|
||||||
deny = any(re.search(p, low) for p in deny_patterns)
|
row = {"agent": aid, "status": "FAIL", "reason": "", "preview": ""}
|
||||||
pos = any(re.search(p, low) for p in pos_patterns)
|
try:
|
||||||
if deny:
|
with urlreq.urlopen(req, timeout=timeout_s) as resp:
|
||||||
row["status"] = "FAIL"
|
data = json.loads(resp.read().decode("utf-8", errors="ignore"))
|
||||||
row["reason"] = "explicit_deny"
|
text = (data.get("response") or "").strip()
|
||||||
elif pos:
|
low = text.lower()
|
||||||
row["status"] = "PASS"
|
row["preview"] = text[:220].replace("\n", " ")
|
||||||
row["reason"] = "role_context_found"
|
|
||||||
else:
|
|
||||||
row["status"] = "FAIL"
|
|
||||||
row["reason"] = "no_role_context"
|
|
||||||
except Exception as e:
|
|
||||||
row["status"] = "FAIL"
|
|
||||||
row["reason"] = f"http_error:{type(e).__name__}"
|
|
||||||
|
|
||||||
rows.append(row)
|
deny = any(re.search(p, low) for p in deny_patterns)
|
||||||
print(f"[{idx:02d}/{len(agents)}] {agent_id}: {row['status']} ({row['reason']})")
|
pos = any(re.search(p, low) for p in pos_patterns)
|
||||||
time.sleep(0.05)
|
if deny:
|
||||||
|
row["status"] = "FAIL"
|
||||||
|
row["reason"] = "explicit_deny"
|
||||||
|
elif pos:
|
||||||
|
row["status"] = "PASS"
|
||||||
|
row["reason"] = "role_context_found"
|
||||||
|
else:
|
||||||
|
row["status"] = "FAIL"
|
||||||
|
row["reason"] = "no_role_context"
|
||||||
|
except Exception as e:
|
||||||
|
row["status"] = "FAIL"
|
||||||
|
row["reason"] = f"http_error:{type(e).__name__}"
|
||||||
|
|
||||||
|
results.append(row)
|
||||||
|
print(f"[{idx:02d}/{len(agents)}] {aid}: {row['status']} ({row['reason']})", flush=True)
|
||||||
|
time.sleep(0.05)
|
||||||
|
else:
|
||||||
|
raise SystemExit(f"[FAIL] unsupported mode: {mode}")
|
||||||
|
|
||||||
summary = {
|
summary = {
|
||||||
"total": len(rows),
|
"mode": mode,
|
||||||
"pass": sum(1 for r in rows if r["status"] == "PASS"),
|
"total": len(results),
|
||||||
"fail": sum(1 for r in rows if r["status"] == "FAIL"),
|
"pass": sum(1 for r in results if r["status"] == "PASS"),
|
||||||
|
"fail": sum(1 for r in results if r["status"] == "FAIL"),
|
||||||
}
|
}
|
||||||
|
payload = {"summary": summary, "results": results}
|
||||||
payload = {"summary": summary, "results": rows}
|
|
||||||
out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
print(json.dumps(summary, ensure_ascii=False))
|
|
||||||
print(f"[INFO] details: {out_json}")
|
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
||||||
|
print(f"[INFO] details: {out_json}", flush=True)
|
||||||
|
|
||||||
if summary["fail"] > 0:
|
if summary["fail"] > 0:
|
||||||
failed = [r["agent"] for r in rows if r["status"] == "FAIL"]
|
failed = [r["agent"] for r in results if r["status"] == "FAIL"]
|
||||||
raise SystemExit(f"[FAIL] DAARWIZZ awareness failed for: {', '.join(failed)}")
|
raise SystemExit(f"[FAIL] DAARWIZZ awareness failed for: {', '.join(failed)}")
|
||||||
|
|
||||||
print("[OK] DAARWIZZ awareness canary passed")
|
print("[OK] DAARWIZZ awareness canary passed", flush=True)
|
||||||
PY
|
PY
|
||||||
|
|||||||
Reference in New Issue
Block a user