microdao-daarion/ops/fabric_preflight.sh

#!/usr/bin/env bash
# Fabric Preflight — verify all nodes before changes/deploys.
# Saves snapshot, compares with previous, fails hard on critical issues.
#
# Usage:
#   bash ops/fabric_preflight.sh [NCS_URL] [NCS_URL2] [ROUTER_URL]
#   bash ops/fabric_preflight.sh  # defaults: 127.0.0.1:8099, same, 127.0.0.1:9102
set -euo pipefail

NODA_NCS="${1:-http://127.0.0.1:8099}"
ROUTER_URL="${2:-http://127.0.0.1:9102}"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
CYAN='\033[0;36m'
NC='\033[0m'

pass()  { echo -e "  ${GREEN}PASS${NC} $1"; }
warn()  { echo -e "  ${YELLOW}WARN${NC} $1"; }
fail()  { echo -e "  ${RED}FAIL${NC} $1"; ERRORS=$((ERRORS+1)); }
info()  { echo -e "  ${CYAN}INFO${NC} $1"; }

ERRORS=0
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SNAPSHOT_DIR="${SCRIPT_DIR}/preflight_snapshots"
mkdir -p "$SNAPSHOT_DIR"

# ── NCS check ─────────────────────────────────────────────────────────────────

check_ncs() {
    local label="$1" url="$2"
    echo "── $label ($url) ──"
    local raw
    raw=$(curl -sf "$url/capabilities" 2>/dev/null) || { fail "NCS unreachable at $url"; return; }

    local node_id served installed swapper_status
    node_id=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('node_id','?'))" 2>/dev/null)
    served=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('served_count',0))" 2>/dev/null)
    installed=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('installed_count',0))" 2>/dev/null)
    swapper_status=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('runtimes',{}).get('swapper',{}).get('status','?'))" 2>/dev/null)

    [ "$served" -gt 0 ] 2>/dev/null && pass "node=$node_id served=$served installed=$installed" \
        || fail "node=$node_id served=$served (empty pool!)"

    [ "$swapper_status" = "disabled" ] && pass "swapper=disabled" || warn "swapper=$swapper_status"

    local caps
    caps=$(echo "$raw" | python3 -c "
import json,sys
c=json.load(sys.stdin).get('capabilities',{})
parts=[f'{k}={v}' for k,v in c.items() if k!='providers']
print(' '.join(parts) if parts else '(none — P3.5 not deployed?)')
" 2>/dev/null)
    [ "$caps" = "(none — P3.5 not deployed?)" ] && warn "capabilities: $caps" || pass "capabilities: $caps"

    local mem_p inflight
    mem_p=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('node_load',{}).get('mem_pressure','?'))" 2>/dev/null)
    inflight=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('node_load',{}).get('inflight',json.load(open('/dev/stdin')).get('node_load',{}).get('inflight_jobs',0)) if False else json.load(sys.stdin).get('node_load',{}).get('inflight_jobs',0))" 2>/dev/null || echo "?")
    [ "$mem_p" = "high" ] && warn "mem_pressure=$mem_p inflight=$inflight" \
        || pass "mem_pressure=$mem_p inflight=$inflight"

    local vision_count
    vision_count=$(echo "$raw" | python3 -c "import json,sys;print(sum(1 for m in json.load(sys.stdin).get('served_models',[]) if m.get('type')=='vision'))" 2>/dev/null)
    [ "$vision_count" -gt 0 ] && pass "vision models: $vision_count" || warn "no vision models served"

    NCS_RAW="$raw"
    NCS_NODE_ID="$node_id"
}

# ── Router check ──────────────────────────────────────────────────────────────

check_router() {
    local label="$1" url="$2"
    echo "── $label ($url) ──"
    local health
    health=$(curl -sf "$url/health" 2>/dev/null) || { fail "Router unreachable at $url"; return; }
    local status
    status=$(echo "$health" | python3 -c "import json,sys;print(json.load(sys.stdin).get('status','?'))" 2>/dev/null)
    [ "$status" = "ok" ] && pass "health=$status" || fail "health=$status"

    local models_total
    models_total=$(curl -sf "$url/v1/models" 2>/dev/null | python3 -c "import json,sys;print(json.load(sys.stdin).get('total',0))" 2>/dev/null) || models_total=0
    [ "$models_total" -gt 0 ] && pass "global pool: $models_total models" || fail "global pool empty"

    local caps_nodes
    caps_nodes=$(curl -sf "$url/v1/capabilities" 2>/dev/null | python3 -c "
import json,sys
d=json.load(sys.stdin)
nodes=list(d.get('capabilities_by_node',{}).keys())
print(f'{len(nodes)} node(s): {\" \".join(nodes)}' if nodes else '(none)')
" 2>/dev/null)
    [ "$caps_nodes" = "(none)" ] && warn "capabilities_by_node: $caps_nodes" || pass "capabilities_by_node: $caps_nodes"

    ROUTER_MODELS=$(curl -sf "$url/v1/models" 2>/dev/null || echo '{}')
}

# ── Snapshot + diff ───────────────────────────────────────────────────────────

NCS_RAW="{}"
NCS_NODE_ID="unknown"
ROUTER_MODELS="{}"

save_and_diff() {
    local ts
    ts=$(date +%Y-%m-%d_%H%M%S)
    local snap_file="${SNAPSHOT_DIR}/${NCS_NODE_ID}_${ts}.json"

    python3 -c "
import json, glob, os, sys
from datetime import datetime

ncs = json.loads('''$(echo "$NCS_RAW" | python3 -c "import sys;print(sys.stdin.read().replace(\"'\",\"\"))")''') if '''$NCS_RAW''' != '{}' else {}
router = json.loads('''$(echo "$ROUTER_MODELS" | python3 -c "import sys;print(sys.stdin.read().replace(\"'\",\"\"))")''') if '''$ROUTER_MODELS''' != '{}' else {}

snapshot = {
    'timestamp': datetime.utcnow().isoformat() + 'Z',
    'node_id': ncs.get('node_id', '$NCS_NODE_ID'),
    'errors': $ERRORS,
    'passed': $ERRORS == 0,
    'served_count': ncs.get('served_count', 0),
    'installed_count': ncs.get('installed_count', 0),
    'capabilities': {k:v for k,v in ncs.get('capabilities', {}).items() if k != 'providers'},
    'providers': ncs.get('capabilities', {}).get('providers', {}),
    'node_load': ncs.get('node_load', {}),
    'router_models_total': router.get('total', 0),
    'capabilities_by_node': router.get('capabilities_by_node', {}),
}

with open('$snap_file', 'w') as f:
    json.dump(snapshot, f, indent=2, ensure_ascii=False)

# Find previous snapshot for diff
prefix = '${NCS_NODE_ID}'.lower() + '_'
snaps = sorted(glob.glob(os.path.join('$SNAPSHOT_DIR', prefix + '*.json')), reverse=True)
prev = None
if len(snaps) >= 2:
    with open(snaps[1]) as f:
        prev = json.load(f)

if prev:
    diffs = []
    for key in ('served_count', 'installed_count', 'router_models_total'):
        ov, nv = prev.get(key, '?'), snapshot.get(key, '?')
        if ov != nv:
            diffs.append(f'  {key}: {ov} → {nv}')
    old_caps = prev.get('capabilities', {})
    new_caps = snapshot.get('capabilities', {})
    for k in sorted(set(list(old_caps.keys()) + list(new_caps.keys()))):
        ov, nv = old_caps.get(k, '?'), new_caps.get(k, '?')
        if ov != nv:
            diffs.append(f'  caps.{k}: {ov} → {nv}')
    if diffs:
        print('Changes vs previous:')
        for d in diffs:
            print(d)
    else:
        print('(no changes vs previous snapshot)')
else:
    print('(first snapshot for this node)')
" 2>/dev/null || echo "(snapshot diff failed)"

    info "Snapshot: $snap_file"
}

# ── Main ──────────────────────────────────────────────────────────────────────

echo "╔══════════════════════════════════════╗"
echo "║   Fabric Preflight Check             ║"
echo "╚══════════════════════════════════════╝"
echo ""

check_ncs "NCS" "$NODA_NCS"
echo ""
check_router "Router" "$ROUTER_URL"
echo ""
save_and_diff
echo ""

if [ $ERRORS -gt 0 ]; then
    echo -e "${RED}Preflight FAILED: $ERRORS error(s)${NC}"
    echo -e "${RED}BLOCKED: no changes allowed until all errors resolved${NC}"
    exit 1
else
    echo -e "${GREEN}Preflight PASSED — changes allowed${NC}"
fi