744c149300
Some checks failed
Build and Deploy Docs / build-and-deploy (push) Has been cancelled
- Created logs/ structure (sessions, operations, incidents) - Added session-start/log/end scripts - Installed Git hooks for auto-logging commits/pushes - Added shell integration for zsh - Created CHANGELOG.md - Documented today's session (2026-01-10)
356 lines
15 KiB
Bash
Executable File
356 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# TRIAGE SCRIPT: Verify if PostgreSQL images are compromised OR NODE1 is compromised
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
#
|
|
# USAGE:
|
|
# ./triage-postgres-compromise.sh [local|remote|compare]
|
|
#
|
|
# MODES:
|
|
# local - Run checks on LOCAL machine (MacBook/clean host)
|
|
# remote - Run checks on NODE1 via SSH
|
|
# compare - Compare results between local and remote
|
|
#
|
|
# REQUIREMENTS:
|
|
# - Docker installed locally
|
|
# - SSH access to NODE1 (root@144.76.224.179)
|
|
# - Run on CLEAN machine (not NODE1!)
|
|
#
|
|
# Created: 2026-01-10
|
|
# Purpose: Incident #4 triage - determine if host or images compromised
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
NODE1_HOST="root@144.76.224.179"
|
|
IMAGES_TO_TEST=(
|
|
"postgres:16-alpine"
|
|
"postgres:16"
|
|
"postgres:15-alpine"
|
|
"postgres:14-alpine"
|
|
"postgres:14"
|
|
)
|
|
|
|
# IOC patterns
|
|
IOC_FILES=(
|
|
"/tmp/httpd"
|
|
"/tmp/.perf.c"
|
|
"/tmp/mysql"
|
|
"/tmp/cpioshuf"
|
|
"/tmp/ipcalc"
|
|
)
|
|
|
|
RESULTS_DIR="/tmp/triage-results-$(date +%Y%m%d-%H%M%S)"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
log_success() { echo -e "${GREEN}[OK]${NC} $1"; }
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# Function: Check container for IOC
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
check_container_ioc() {
|
|
local image="$1"
|
|
local location="$2" # "local" or "remote"
|
|
local result_file="$RESULTS_DIR/${image//[:\/]/_}_${location}.txt"
|
|
|
|
log_info "Testing $image on $location..."
|
|
|
|
local docker_cmd="docker"
|
|
if [[ "$location" == "remote" ]]; then
|
|
docker_cmd="ssh $NODE1_HOST docker"
|
|
fi
|
|
|
|
# Pull image
|
|
$docker_cmd pull "$image" 2>/dev/null || {
|
|
log_error "Failed to pull $image"
|
|
echo "PULL_FAILED" > "$result_file"
|
|
return 1
|
|
}
|
|
|
|
# Get digest
|
|
local digest=$($docker_cmd inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null || echo "NO_DIGEST")
|
|
echo "DIGEST: $digest" > "$result_file"
|
|
|
|
# Run container and check /tmp
|
|
local tmp_contents=$($docker_cmd run --rm "$image" sh -c "ls -la /tmp/ 2>/dev/null; find /tmp -type f 2>/dev/null" 2>/dev/null || echo "RUN_FAILED")
|
|
echo "TMP_CONTENTS:" >> "$result_file"
|
|
echo "$tmp_contents" >> "$result_file"
|
|
|
|
# Check for IOC
|
|
local ioc_found=0
|
|
for ioc in "${IOC_FILES[@]}"; do
|
|
if echo "$tmp_contents" | grep -q "$(basename "$ioc")"; then
|
|
log_error "IOC FOUND in $image on $location: $ioc"
|
|
echo "IOC_FOUND: $ioc" >> "$result_file"
|
|
ioc_found=1
|
|
fi
|
|
done
|
|
|
|
if [[ $ioc_found -eq 0 ]]; then
|
|
log_success "$image on $location: CLEAN"
|
|
echo "STATUS: CLEAN" >> "$result_file"
|
|
else
|
|
log_error "$image on $location: INFECTED"
|
|
echo "STATUS: INFECTED" >> "$result_file"
|
|
fi
|
|
|
|
return $ioc_found
|
|
}
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# Function: Check NODE1 host for persistence
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
check_node1_persistence() {
|
|
log_info "Checking NODE1 for persistence mechanisms..."
|
|
local result_file="$RESULTS_DIR/node1_persistence.txt"
|
|
|
|
ssh "$NODE1_HOST" << 'REMOTE_SCRIPT' > "$result_file" 2>&1
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "NODE1 PERSISTENCE CHECK - $(date)"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
|
|
echo ""
|
|
echo "=== CRON JOBS ==="
|
|
echo "--- root crontab ---"
|
|
crontab -l 2>/dev/null || echo "(empty)"
|
|
echo "--- /etc/crontab ---"
|
|
cat /etc/crontab 2>/dev/null | grep -v "^#" | grep -v "^$"
|
|
echo "--- /etc/cron.d/ ---"
|
|
ls -la /etc/cron.d/ 2>/dev/null
|
|
for f in /etc/cron.d/*; do
|
|
echo "--- $f ---"
|
|
cat "$f" 2>/dev/null | grep -v "^#" | grep -v "^$"
|
|
done
|
|
|
|
echo ""
|
|
echo "=== SYSTEMD SERVICES (suspicious) ==="
|
|
systemctl list-units --type=service --all 2>/dev/null | grep -iE "perf|miner|http|crypto|kdev|kinsin" || echo "(none found)"
|
|
|
|
echo ""
|
|
echo "=== LD_PRELOAD ==="
|
|
echo "--- /etc/ld.so.preload ---"
|
|
cat /etc/ld.so.preload 2>/dev/null || echo "(not exists)"
|
|
echo "--- LD_PRELOAD env ---"
|
|
echo "${LD_PRELOAD:-not set}"
|
|
|
|
echo ""
|
|
echo "=== SUSPICIOUS PROCESSES ==="
|
|
ps aux | grep -E "(httpd|xmrig|kdevtmp|kinsing|perfctl|\.perf|softirq|vrarhpb)" | grep -v grep || echo "(none running)"
|
|
|
|
echo ""
|
|
echo "=== HIGH CPU PROCESSES ==="
|
|
ps aux --sort=-%cpu | head -10
|
|
|
|
echo ""
|
|
echo "=== NETWORK CONNECTIONS (mining pools) ==="
|
|
ss -anp 2>/dev/null | grep -E "(3333|4444|5555|8080|8888|14433|14444)" | head -20 || echo "(none found)"
|
|
|
|
echo ""
|
|
echo "=== SSH AUTHORIZED KEYS ==="
|
|
echo "--- /root/.ssh/authorized_keys ---"
|
|
cat /root/.ssh/authorized_keys 2>/dev/null | head -20
|
|
|
|
echo ""
|
|
echo "=== /tmp CONTENTS ==="
|
|
ls -la /tmp/ 2>/dev/null
|
|
find /tmp -type f -executable 2>/dev/null
|
|
|
|
echo ""
|
|
echo "=== /var/tmp CONTENTS ==="
|
|
find /var/tmp -type f -executable 2>/dev/null
|
|
|
|
echo ""
|
|
echo "=== /dev/shm CONTENTS ==="
|
|
ls -la /dev/shm/ 2>/dev/null
|
|
|
|
echo ""
|
|
echo "=== DOCKER DAEMON CONFIG ==="
|
|
cat /etc/docker/daemon.json 2>/dev/null || echo "(default config)"
|
|
|
|
echo ""
|
|
echo "=== KERNEL MODULES (first 30) ==="
|
|
lsmod | head -30
|
|
|
|
echo ""
|
|
echo "=== SYSTEM LOAD ==="
|
|
uptime
|
|
cat /proc/loadavg
|
|
|
|
echo ""
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "CHECK COMPLETE"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
REMOTE_SCRIPT
|
|
|
|
log_info "Results saved to: $result_file"
|
|
|
|
# Quick analysis
|
|
if grep -qE "(httpd|xmrig|perfctl|kdevtmp|kinsing)" "$result_file"; then
|
|
log_error "SUSPICIOUS ACTIVITY DETECTED ON NODE1!"
|
|
return 1
|
|
else
|
|
log_success "No obvious persistence found (manual review recommended)"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# Function: Compare local vs remote results
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
compare_results() {
|
|
log_info "Comparing local vs remote results..."
|
|
|
|
echo ""
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "COMPARISON RESULTS"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
|
|
local local_infected=0
|
|
local remote_infected=0
|
|
|
|
for image in "${IMAGES_TO_TEST[@]}"; do
|
|
local safe_name="${image//[:\/]/_}"
|
|
local local_file="$RESULTS_DIR/${safe_name}_local.txt"
|
|
local remote_file="$RESULTS_DIR/${safe_name}_remote.txt"
|
|
|
|
echo "Image: $image"
|
|
echo " Local digest: $(grep "DIGEST:" "$local_file" 2>/dev/null | cut -d' ' -f2- || echo "N/A")"
|
|
echo " Remote digest: $(grep "DIGEST:" "$remote_file" 2>/dev/null | cut -d' ' -f2- || echo "N/A")"
|
|
echo " Local status: $(grep "STATUS:" "$local_file" 2>/dev/null | cut -d' ' -f2 || echo "N/A")"
|
|
echo " Remote status: $(grep "STATUS:" "$remote_file" 2>/dev/null | cut -d' ' -f2 || echo "N/A")"
|
|
|
|
if grep -q "STATUS: INFECTED" "$local_file" 2>/dev/null; then
|
|
local_infected=$((local_infected + 1))
|
|
fi
|
|
if grep -q "STATUS: INFECTED" "$remote_file" 2>/dev/null; then
|
|
remote_infected=$((remote_infected + 1))
|
|
fi
|
|
echo ""
|
|
done
|
|
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "VERDICT:"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
|
|
if [[ $local_infected -eq 0 && $remote_infected -gt 0 ]]; then
|
|
echo -e "${RED}🔴 NODE1 IS COMPROMISED${NC}"
|
|
echo ""
|
|
echo "Evidence: Same images are CLEAN locally but INFECTED on NODE1"
|
|
echo ""
|
|
echo "REQUIRED ACTIONS:"
|
|
echo " 1. STOP using NODE1 immediately"
|
|
echo " 2. Rotate ALL secrets (SSH keys, tokens, passwords)"
|
|
echo " 3. Full OS reinstall (not cleanup!)"
|
|
echo " 4. Deploy only verified images from clean host"
|
|
elif [[ $local_infected -gt 0 && $remote_infected -gt 0 ]]; then
|
|
echo -e "${RED}🔴 DOCKER HUB IMAGES MAY BE COMPROMISED${NC}"
|
|
echo ""
|
|
echo "Evidence: Same images are INFECTED both locally and on NODE1"
|
|
echo ""
|
|
echo "REQUIRED ACTIONS:"
|
|
echo " 1. Report to Docker Security team"
|
|
echo " 2. Use alternative registry (GHCR, Quay.io)"
|
|
echo " 3. Build from source"
|
|
echo " 4. Scan all images with Trivy before use"
|
|
elif [[ $local_infected -eq 0 && $remote_infected -eq 0 ]]; then
|
|
echo -e "${GREEN}✅ ALL IMAGES APPEAR CLEAN${NC}"
|
|
echo ""
|
|
echo "However, if you saw IOC earlier, the malware may be:"
|
|
echo " - Triggered by specific conditions"
|
|
echo " - Using time-based activation"
|
|
echo " - Detected your testing and hiding"
|
|
echo ""
|
|
echo "Recommend: Still perform NODE1 persistence check"
|
|
else
|
|
echo -e "${YELLOW}⚠️ INCONCLUSIVE RESULTS${NC}"
|
|
echo ""
|
|
echo "Manual investigation required"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Full results saved in: $RESULTS_DIR"
|
|
}
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# Main
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
main() {
|
|
local mode="${1:-help}"
|
|
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "PostgreSQL Compromise Triage Script"
|
|
echo "Results directory: $RESULTS_DIR"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
|
|
case "$mode" in
|
|
local)
|
|
log_info "Running LOCAL checks (this machine)..."
|
|
for image in "${IMAGES_TO_TEST[@]}"; do
|
|
check_container_ioc "$image" "local" || true
|
|
done
|
|
;;
|
|
remote)
|
|
log_info "Running REMOTE checks (NODE1)..."
|
|
check_node1_persistence || true
|
|
for image in "${IMAGES_TO_TEST[@]}"; do
|
|
check_container_ioc "$image" "remote" || true
|
|
done
|
|
;;
|
|
compare)
|
|
log_info "Running FULL comparison..."
|
|
log_warn "This will pull images on both local and NODE1"
|
|
echo ""
|
|
|
|
# Run local checks
|
|
for image in "${IMAGES_TO_TEST[@]}"; do
|
|
check_container_ioc "$image" "local" || true
|
|
done
|
|
|
|
# Run remote checks
|
|
check_node1_persistence || true
|
|
for image in "${IMAGES_TO_TEST[@]}"; do
|
|
check_container_ioc "$image" "remote" || true
|
|
done
|
|
|
|
# Compare
|
|
compare_results
|
|
;;
|
|
persistence)
|
|
check_node1_persistence
|
|
cat "$RESULTS_DIR/node1_persistence.txt"
|
|
;;
|
|
help|*)
|
|
echo "Usage: $0 [local|remote|compare|persistence]"
|
|
echo ""
|
|
echo "Modes:"
|
|
echo " local - Check images on THIS machine (should be clean)"
|
|
echo " remote - Check images on NODE1 + persistence mechanisms"
|
|
echo " compare - Run both and compare results (RECOMMENDED)"
|
|
echo " persistence - Only check NODE1 for persistence mechanisms"
|
|
echo ""
|
|
echo "⚠️ Run this script from a CLEAN machine (not NODE1!)"
|
|
echo ""
|
|
echo "Example workflow:"
|
|
echo " 1. ./triage-postgres-compromise.sh compare"
|
|
echo " 2. Review results in $RESULTS_DIR"
|
|
echo " 3. If NODE1 compromised → full rebuild"
|
|
echo " 4. If images compromised → report to Docker"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|