microdao-daarion/ops/status.sh

#!/bin/bash
#
# NODE1 Status Check Script
# Version: 1.1
# Last Updated: 2026-01-26
#
# Usage: ./ops/status.sh [--remote] [--verbose]
#   --remote   Run on NODE1 via SSH
#   --verbose  Show detailed output
#

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Config
NODE1_HOST="144.76.224.179"
NODE1_USER="root"

# Parse arguments
REMOTE=false
VERBOSE=false
for arg in "$@"; do
    case $arg in
        --remote) REMOTE=true ;;
        --verbose) VERBOSE=true ;;
    esac
done

# Remote execution wrapper
if [ "$REMOTE" = true ]; then
    echo "Connecting to NODE1 ($NODE1_HOST)..."
    
    # Copy and execute on remote
    ssh -o StrictHostKeyChecking=accept-new ${NODE1_USER}@${NODE1_HOST} "bash -s" << 'REMOTE_SCRIPT'
# Inline the status check for remote execution
set -e

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

echo "========================================"
echo "  NODE1 Status Check (Remote)"
echo "  $(date '+%Y-%m-%d %H:%M:%S')"
echo "========================================"
echo ""

echo "=== Health Endpoints ==="
failed=0

for check in "Router:9102:/health" "Gateway:9300:/health" "Memory:8000:/health" \
             "RAG:9500:/health" "Swapper:8890:/health" "Qdrant:6333:/healthz" \
             "Vision:8001:/health" "Parser:8101:/health" "Prometheus:9090:/-/healthy" \
             "Grafana:3030:/api/health"; do
    name="${check%%:*}"
    rest="${check#*:}"
    port="${rest%%:*}"
    path="${rest#*:}"
    url="http://127.0.0.1:${port}${path}"
    
    code=$(curl -sS -m 5 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000")
    
    if [ "$code" = "200" ]; then
        echo -e "${GREEN}✅${NC} $name (${port}): ${GREEN}$code${NC}"
    else
        echo -e "${RED}❌${NC} $name (${port}): ${RED}$code${NC}"
        ((failed++)) || true
    fi
done

echo ""
echo "=== DNS Resolution ==="
if docker exec dagi-memory-service-node1 python3 -c "import socket; socket.gethostbyname('dagi-qdrant-node1')" 2>/dev/null; then
    echo -e "${GREEN}✅${NC} DNS: dagi-qdrant-node1 resolves"
else
    echo -e "${RED}❌${NC} DNS: dagi-qdrant-node1 does NOT resolve"
fi

echo ""
echo "=== System Resources ==="
echo "Hostname: $(hostname)"
echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
echo "Memory: $(free -h | awk '/^Mem:/ {print $3 "/" $2}')"
echo "Disk: $(df -h / | awk 'NR==2 {print $3 "/" $2 " (" $5 " used)"}')"

echo ""
echo "=== Docker Containers (top 15) ==="
docker ps --format "table {{.Names}}\t{{.Status}}" | head -16

echo ""
echo "========================================"

REMOTE_SCRIPT

else
    # Local execution (for running on NODE1 directly)
    echo "========================================"
    echo "  NODE1 Status Check (Local)"
    echo "  $(date '+%Y-%m-%d %H:%M:%S')"
    echo "========================================"
    echo ""
    
    echo "=== Health Endpoints ==="
    failed=0
    
    for check in "Router:9102:/health" "Gateway:9300:/health" "Memory:8000:/health" \
                 "RAG:9500:/health" "Swapper:8890:/health" "Qdrant:6333:/healthz" \
                 "Vision:8001:/health" "Parser:8101:/health" "Prometheus:9090:/-/healthy" \
                 "Grafana:3030:/api/health"; do
        name="${check%%:*}"
        rest="${check#*:}"
        port="${rest%%:*}"
        path="${rest#*:}"
        url="http://127.0.0.1:${port}${path}"
        
        code=$(curl -sS -m 5 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000")
        
        if [ "$code" = "200" ]; then
            echo -e "${GREEN}✅${NC} $name (${port}): ${GREEN}$code${NC}"
        else
            echo -e "${RED}❌${NC} $name (${port}): ${RED}$code${NC}"
            ((failed++)) || true
        fi
    done
    
    echo ""
    echo "=== DNS Resolution ==="
    if docker exec dagi-memory-service-node1 python3 -c "import socket; socket.gethostbyname('dagi-qdrant-node1')" 2>/dev/null; then
        echo -e "${GREEN}✅${NC} DNS: dagi-qdrant-node1 resolves"
    else
        echo -e "${RED}❌${NC} DNS: dagi-qdrant-node1 does NOT resolve"
    fi
    
    echo ""
    echo "=== System Resources ==="
    echo "Hostname: $(hostname)"
    echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
    echo "Memory: $(free -h | awk '/^Mem:/ {print $3 "/" $2}')"
    echo "Disk: $(df -h / | awk 'NR==2 {print $3 "/" $2 " (" $5 " used)"}')"
    
    if [ "$VERBOSE" = true ]; then
        echo ""
        echo "=== Docker Containers ==="
        docker ps --format "table {{.Names}}\t{{.Status}}" | head -20
        
        echo ""
        echo "=== SECURITY CHECKS ==="
        sec_ok=0
        sec_fail=0
        
        # Check 1: Internal ports blocked by iptables
        echo -n "Internal ports blocked: "
        ipt_blocked=$(iptables -L DOCKER-USER -n 2>/dev/null | grep -cE "DROP.*dpt:(9300|9102|6333|9090|3030|8890|8000|9500|8001|8101)" || echo "0")
        if [ "$ipt_blocked" -ge 5 ]; then
            echo -e "${GREEN}OK${NC} (iptables: $ipt_blocked rules)"
            ((sec_ok++)) || true
        else
            echo -e "${RED}FAIL${NC} (iptables: $ipt_blocked rules)"
            ((sec_fail++)) || true
        fi
        
        # Check 2: HSTS header present
        echo -n "HSTS header: "
        hsts=$(curl -sS -k -I https://127.0.0.1/ 2>/dev/null | grep -i "strict-transport-security" | wc -l)
        if [ "$hsts" -gt 0 ]; then
            echo -e "${GREEN}OK${NC}"
            ((sec_ok++)) || true
        else
            echo -e "${RED}FAIL${NC}"
            ((sec_fail++)) || true
        fi
        
        # Check 3: WAF blocks .env
        echo -n "WAF blocks .env: "
        waf_code=$(curl -sS -k -o /dev/null -w "%{http_code}" --max-time 3 https://127.0.0.1/.env 2>&1 | tail -c 3)
        if [ "$waf_code" = "000" ] || [ "$waf_code" = "444" ] || [ "$waf_code" = "403" ] || [ -z "$waf_code" ]; then
            echo -e "${GREEN}OK${NC} (blocked)"
            ((sec_ok++)) || true
        else
            echo -e "${RED}FAIL${NC} ($waf_code)"
            ((sec_fail++)) || true
        fi
        
        # Check 4: iptables DOCKER-USER rules present
        echo -n "iptables DOCKER-USER: "
        ipt_rules=$(iptables -L DOCKER-USER -n 2>/dev/null | grep -c "DROP.*dpt:" || echo "0")
        if [ "$ipt_rules" -ge 5 ]; then
            echo -e "${GREEN}OK${NC} ($ipt_rules rules)"
            ((sec_ok++)) || true
        else
            echo -e "${RED}FAIL${NC} ($ipt_rules rules)"
            ((sec_fail++)) || true
        fi
        
        # Check 5: Nginx running
        echo -n "Nginx status: "
        if systemctl is-active nginx >/dev/null 2>&1; then
            echo -e "${GREEN}OK${NC}"
            ((sec_ok++)) || true
        else
            echo -e "${RED}FAIL${NC}"
            ((sec_fail++)) || true
        fi
        
        # Check 6: HTTP→HTTPS redirect
        echo -n "HTTP→HTTPS redirect: "
        http_code=$(curl -sS -o /dev/null -w "%{http_code}" http://127.0.0.1/ 2>/dev/null || echo "000")
        if [ "$http_code" = "301" ]; then
            echo -e "${GREEN}OK${NC}"
            ((sec_ok++)) || true
        else
            echo -e "${YELLOW}WARN${NC} ($http_code)"
            ((sec_fail++)) || true
        fi
        
        echo ""
        echo "Security: $sec_ok passed, $sec_fail failed"
    fi
    
    echo ""
    echo "========================================"
fi