✨ Add automated session logging system
Some checks failed
Build and Deploy Docs / build-and-deploy (push) Has been cancelled
Some checks failed
Build and Deploy Docs / build-and-deploy (push) Has been cancelled
- Created logs/ structure (sessions, operations, incidents) - Added session-start/log/end scripts - Installed Git hooks for auto-logging commits/pushes - Added shell integration for zsh - Created CHANGELOG.md - Documented today's session (2026-01-10)
This commit is contained in:
Apple
355
scripts/security/triage-postgres-compromise.sh
Executable file
355
scripts/security/triage-postgres-compromise.sh
Executable file
@@ -0,0 +1,355 @@
|
||||
#!/bin/bash
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# TRIAGE SCRIPT: Verify if PostgreSQL images are compromised OR NODE1 is compromised
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# USAGE:
|
||||
# ./triage-postgres-compromise.sh [local|remote|compare]
|
||||
#
|
||||
# MODES:
|
||||
# local - Run checks on LOCAL machine (MacBook/clean host)
|
||||
# remote - Run checks on NODE1 via SSH
|
||||
# compare - Compare results between local and remote
|
||||
#
|
||||
# REQUIREMENTS:
|
||||
# - Docker installed locally
|
||||
# - SSH access to NODE1 (root@144.76.224.179)
|
||||
# - Run on CLEAN machine (not NODE1!)
|
||||
#
|
||||
# Created: 2026-01-10
|
||||
# Purpose: Incident #4 triage - determine if host or images compromised
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
NODE1_HOST="root@144.76.224.179"
|
||||
IMAGES_TO_TEST=(
|
||||
"postgres:16-alpine"
|
||||
"postgres:16"
|
||||
"postgres:15-alpine"
|
||||
"postgres:14-alpine"
|
||||
"postgres:14"
|
||||
)
|
||||
|
||||
# IOC patterns
|
||||
IOC_FILES=(
|
||||
"/tmp/httpd"
|
||||
"/tmp/.perf.c"
|
||||
"/tmp/mysql"
|
||||
"/tmp/cpioshuf"
|
||||
"/tmp/ipcalc"
|
||||
)
|
||||
|
||||
RESULTS_DIR="/tmp/triage-results-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
log_success() { echo -e "${GREEN}[OK]${NC} $1"; }
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Function: Check container for IOC
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
check_container_ioc() {
|
||||
local image="$1"
|
||||
local location="$2" # "local" or "remote"
|
||||
local result_file="$RESULTS_DIR/${image//[:\/]/_}_${location}.txt"
|
||||
|
||||
log_info "Testing $image on $location..."
|
||||
|
||||
local docker_cmd="docker"
|
||||
if [[ "$location" == "remote" ]]; then
|
||||
docker_cmd="ssh $NODE1_HOST docker"
|
||||
fi
|
||||
|
||||
# Pull image
|
||||
$docker_cmd pull "$image" 2>/dev/null || {
|
||||
log_error "Failed to pull $image"
|
||||
echo "PULL_FAILED" > "$result_file"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Get digest
|
||||
local digest=$($docker_cmd inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null || echo "NO_DIGEST")
|
||||
echo "DIGEST: $digest" > "$result_file"
|
||||
|
||||
# Run container and check /tmp
|
||||
local tmp_contents=$($docker_cmd run --rm "$image" sh -c "ls -la /tmp/ 2>/dev/null; find /tmp -type f 2>/dev/null" 2>/dev/null || echo "RUN_FAILED")
|
||||
echo "TMP_CONTENTS:" >> "$result_file"
|
||||
echo "$tmp_contents" >> "$result_file"
|
||||
|
||||
# Check for IOC
|
||||
local ioc_found=0
|
||||
for ioc in "${IOC_FILES[@]}"; do
|
||||
if echo "$tmp_contents" | grep -q "$(basename "$ioc")"; then
|
||||
log_error "IOC FOUND in $image on $location: $ioc"
|
||||
echo "IOC_FOUND: $ioc" >> "$result_file"
|
||||
ioc_found=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $ioc_found -eq 0 ]]; then
|
||||
log_success "$image on $location: CLEAN"
|
||||
echo "STATUS: CLEAN" >> "$result_file"
|
||||
else
|
||||
log_error "$image on $location: INFECTED"
|
||||
echo "STATUS: INFECTED" >> "$result_file"
|
||||
fi
|
||||
|
||||
return $ioc_found
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Function: Check NODE1 host for persistence
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
check_node1_persistence() {
|
||||
log_info "Checking NODE1 for persistence mechanisms..."
|
||||
local result_file="$RESULTS_DIR/node1_persistence.txt"
|
||||
|
||||
ssh "$NODE1_HOST" << 'REMOTE_SCRIPT' > "$result_file" 2>&1
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo "NODE1 PERSISTENCE CHECK - $(date)"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
|
||||
echo ""
|
||||
echo "=== CRON JOBS ==="
|
||||
echo "--- root crontab ---"
|
||||
crontab -l 2>/dev/null || echo "(empty)"
|
||||
echo "--- /etc/crontab ---"
|
||||
cat /etc/crontab 2>/dev/null | grep -v "^#" | grep -v "^$"
|
||||
echo "--- /etc/cron.d/ ---"
|
||||
ls -la /etc/cron.d/ 2>/dev/null
|
||||
for f in /etc/cron.d/*; do
|
||||
echo "--- $f ---"
|
||||
cat "$f" 2>/dev/null | grep -v "^#" | grep -v "^$"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== SYSTEMD SERVICES (suspicious) ==="
|
||||
systemctl list-units --type=service --all 2>/dev/null | grep -iE "perf|miner|http|crypto|kdev|kinsin" || echo "(none found)"
|
||||
|
||||
echo ""
|
||||
echo "=== LD_PRELOAD ==="
|
||||
echo "--- /etc/ld.so.preload ---"
|
||||
cat /etc/ld.so.preload 2>/dev/null || echo "(not exists)"
|
||||
echo "--- LD_PRELOAD env ---"
|
||||
echo "${LD_PRELOAD:-not set}"
|
||||
|
||||
echo ""
|
||||
echo "=== SUSPICIOUS PROCESSES ==="
|
||||
ps aux | grep -E "(httpd|xmrig|kdevtmp|kinsing|perfctl|\.perf|softirq|vrarhpb)" | grep -v grep || echo "(none running)"
|
||||
|
||||
echo ""
|
||||
echo "=== HIGH CPU PROCESSES ==="
|
||||
ps aux --sort=-%cpu | head -10
|
||||
|
||||
echo ""
|
||||
echo "=== NETWORK CONNECTIONS (mining pools) ==="
|
||||
ss -anp 2>/dev/null | grep -E "(3333|4444|5555|8080|8888|14433|14444)" | head -20 || echo "(none found)"
|
||||
|
||||
echo ""
|
||||
echo "=== SSH AUTHORIZED KEYS ==="
|
||||
echo "--- /root/.ssh/authorized_keys ---"
|
||||
cat /root/.ssh/authorized_keys 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== /tmp CONTENTS ==="
|
||||
ls -la /tmp/ 2>/dev/null
|
||||
find /tmp -type f -executable 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== /var/tmp CONTENTS ==="
|
||||
find /var/tmp -type f -executable 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== /dev/shm CONTENTS ==="
|
||||
ls -la /dev/shm/ 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== DOCKER DAEMON CONFIG ==="
|
||||
cat /etc/docker/daemon.json 2>/dev/null || echo "(default config)"
|
||||
|
||||
echo ""
|
||||
echo "=== KERNEL MODULES (first 30) ==="
|
||||
lsmod | head -30
|
||||
|
||||
echo ""
|
||||
echo "=== SYSTEM LOAD ==="
|
||||
uptime
|
||||
cat /proc/loadavg
|
||||
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo "CHECK COMPLETE"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
REMOTE_SCRIPT
|
||||
|
||||
log_info "Results saved to: $result_file"
|
||||
|
||||
# Quick analysis
|
||||
if grep -qE "(httpd|xmrig|perfctl|kdevtmp|kinsing)" "$result_file"; then
|
||||
log_error "SUSPICIOUS ACTIVITY DETECTED ON NODE1!"
|
||||
return 1
|
||||
else
|
||||
log_success "No obvious persistence found (manual review recommended)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Function: Compare local vs remote results
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
compare_results() {
|
||||
log_info "Comparing local vs remote results..."
|
||||
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo "COMPARISON RESULTS"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
local local_infected=0
|
||||
local remote_infected=0
|
||||
|
||||
for image in "${IMAGES_TO_TEST[@]}"; do
|
||||
local safe_name="${image//[:\/]/_}"
|
||||
local local_file="$RESULTS_DIR/${safe_name}_local.txt"
|
||||
local remote_file="$RESULTS_DIR/${safe_name}_remote.txt"
|
||||
|
||||
echo "Image: $image"
|
||||
echo " Local digest: $(grep "DIGEST:" "$local_file" 2>/dev/null | cut -d' ' -f2- || echo "N/A")"
|
||||
echo " Remote digest: $(grep "DIGEST:" "$remote_file" 2>/dev/null | cut -d' ' -f2- || echo "N/A")"
|
||||
echo " Local status: $(grep "STATUS:" "$local_file" 2>/dev/null | cut -d' ' -f2 || echo "N/A")"
|
||||
echo " Remote status: $(grep "STATUS:" "$remote_file" 2>/dev/null | cut -d' ' -f2 || echo "N/A")"
|
||||
|
||||
if grep -q "STATUS: INFECTED" "$local_file" 2>/dev/null; then
|
||||
local_infected=$((local_infected + 1))
|
||||
fi
|
||||
if grep -q "STATUS: INFECTED" "$remote_file" 2>/dev/null; then
|
||||
remote_infected=$((remote_infected + 1))
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo "VERDICT:"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
|
||||
if [[ $local_infected -eq 0 && $remote_infected -gt 0 ]]; then
|
||||
echo -e "${RED}🔴 NODE1 IS COMPROMISED${NC}"
|
||||
echo ""
|
||||
echo "Evidence: Same images are CLEAN locally but INFECTED on NODE1"
|
||||
echo ""
|
||||
echo "REQUIRED ACTIONS:"
|
||||
echo " 1. STOP using NODE1 immediately"
|
||||
echo " 2. Rotate ALL secrets (SSH keys, tokens, passwords)"
|
||||
echo " 3. Full OS reinstall (not cleanup!)"
|
||||
echo " 4. Deploy only verified images from clean host"
|
||||
elif [[ $local_infected -gt 0 && $remote_infected -gt 0 ]]; then
|
||||
echo -e "${RED}🔴 DOCKER HUB IMAGES MAY BE COMPROMISED${NC}"
|
||||
echo ""
|
||||
echo "Evidence: Same images are INFECTED both locally and on NODE1"
|
||||
echo ""
|
||||
echo "REQUIRED ACTIONS:"
|
||||
echo " 1. Report to Docker Security team"
|
||||
echo " 2. Use alternative registry (GHCR, Quay.io)"
|
||||
echo " 3. Build from source"
|
||||
echo " 4. Scan all images with Trivy before use"
|
||||
elif [[ $local_infected -eq 0 && $remote_infected -eq 0 ]]; then
|
||||
echo -e "${GREEN}✅ ALL IMAGES APPEAR CLEAN${NC}"
|
||||
echo ""
|
||||
echo "However, if you saw IOC earlier, the malware may be:"
|
||||
echo " - Triggered by specific conditions"
|
||||
echo " - Using time-based activation"
|
||||
echo " - Detected your testing and hiding"
|
||||
echo ""
|
||||
echo "Recommend: Still perform NODE1 persistence check"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ INCONCLUSIVE RESULTS${NC}"
|
||||
echo ""
|
||||
echo "Manual investigation required"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Full results saved in: $RESULTS_DIR"
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Main
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
main() {
|
||||
local mode="${1:-help}"
|
||||
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo "PostgreSQL Compromise Triage Script"
|
||||
echo "Results directory: $RESULTS_DIR"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
case "$mode" in
|
||||
local)
|
||||
log_info "Running LOCAL checks (this machine)..."
|
||||
for image in "${IMAGES_TO_TEST[@]}"; do
|
||||
check_container_ioc "$image" "local" || true
|
||||
done
|
||||
;;
|
||||
remote)
|
||||
log_info "Running REMOTE checks (NODE1)..."
|
||||
check_node1_persistence || true
|
||||
for image in "${IMAGES_TO_TEST[@]}"; do
|
||||
check_container_ioc "$image" "remote" || true
|
||||
done
|
||||
;;
|
||||
compare)
|
||||
log_info "Running FULL comparison..."
|
||||
log_warn "This will pull images on both local and NODE1"
|
||||
echo ""
|
||||
|
||||
# Run local checks
|
||||
for image in "${IMAGES_TO_TEST[@]}"; do
|
||||
check_container_ioc "$image" "local" || true
|
||||
done
|
||||
|
||||
# Run remote checks
|
||||
check_node1_persistence || true
|
||||
for image in "${IMAGES_TO_TEST[@]}"; do
|
||||
check_container_ioc "$image" "remote" || true
|
||||
done
|
||||
|
||||
# Compare
|
||||
compare_results
|
||||
;;
|
||||
persistence)
|
||||
check_node1_persistence
|
||||
cat "$RESULTS_DIR/node1_persistence.txt"
|
||||
;;
|
||||
help|*)
|
||||
echo "Usage: $0 [local|remote|compare|persistence]"
|
||||
echo ""
|
||||
echo "Modes:"
|
||||
echo " local - Check images on THIS machine (should be clean)"
|
||||
echo " remote - Check images on NODE1 + persistence mechanisms"
|
||||
echo " compare - Run both and compare results (RECOMMENDED)"
|
||||
echo " persistence - Only check NODE1 for persistence mechanisms"
|
||||
echo ""
|
||||
echo "⚠️ Run this script from a CLEAN machine (not NODE1!)"
|
||||
echo ""
|
||||
echo "Example workflow:"
|
||||
echo " 1. ./triage-postgres-compromise.sh compare"
|
||||
echo " 2. Review results in $RESULTS_DIR"
|
||||
echo " 3. If NODE1 compromised → full rebuild"
|
||||
echo " 4. If images compromised → report to Docker"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user