microdao-daarion/services/integration-service/src/dlq.ts

import fs from "fs";
import path from "path";
import Redis from "ioredis";
import { logger } from "./logger";
import crypto from "crypto";

const enabled = (process.env.DLQ_ENABLED || "true") === "true";
const redisUrl = process.env.REDIS_URL;
const dlqFile = process.env.DLQ_FILE || "/data/dlq.jsonl";
let redis: Redis | null = null;

const MAX_BYTES = Number(process.env.AGX_AUDIT_MAX_BYTES || 4096);
const REDACT_KEYS = new Set((process.env.AGX_AUDIT_REDACT_KEYS || "token,secret,password,authorization,cookie,api_key,signature").split(",").map(s => s.trim().toLowerCase()));

function sanitize(obj: any): any {
  if (Array.isArray(obj)) return obj.map(sanitize);
  if (obj && typeof obj === "object") {
    const out: any = {};
    for (const [k, v] of Object.entries(obj)) {
      if (REDACT_KEYS.has(k.toLowerCase())) {
        out[k] = "***REDACTED***";
      } else {
        out[k] = sanitize(v);
      }
    }
    return out;
  }
  return obj;
}

function previewPayload(payload: any) {
  const sanitized = sanitize(payload);
  const raw = JSON.stringify(sanitized);
  const size = Buffer.byteLength(raw, "utf-8");
  if (size > MAX_BYTES) {
    return { preview: raw.slice(0, MAX_BYTES), truncated: true, size };
  }
  return { preview: raw, truncated: false, size };
}

if (redisUrl) {
  redis = new Redis(redisUrl);
}

export async function enqueue(type: string, payload: any, meta: any) {
  if (!enabled) return;
  const { preview, truncated, size } = previewPayload(payload);
  const entry = {
    type,
    trace_id: meta?.trace_id || "",
    request_id: meta?.request_id || "",
    route: meta?.route || "",
    dedup_key: meta?.dedup_key || "",
    error_class: meta?.error_class || "",
    error_message: meta?.error_message || "",
    ingest_ts: new Date().toISOString(),
    payload_size: size,
    payload_hash: `sha256:${crypto.createHash("sha256").update(preview).digest("hex")}` ,
    payload_truncated: truncated,
    payload: preview
  };
  if (redis) {
    await redis.xadd("agx:dlq", "*", "data", JSON.stringify(entry));
    return;
  }
  fs.mkdirSync(path.dirname(dlqFile), { recursive: true });
  fs.appendFileSync(dlqFile, JSON.stringify(entry) + "
");
}

export async function replay(handler: (entry: any) => Promise<void>) {
  if (redis) {
    const entries = await redis.xrange("agx:dlq", "-", "+");
    for (const [_id, fields] of entries) {
      const dataIdx = fields.findIndex((v) => v === "data");
      const raw = dataIdx >= 0 ? fields[dataIdx + 1] : "";
      if (!raw) continue;
      const entry = JSON.parse(raw);
      await handler(entry);
    }
    await redis.del("agx:dlq");
    return;
  }
  if (!fs.existsSync(dlqFile)) return;
  const lines = fs.readFileSync(dlqFile, "utf-8").trim().split("
");
  fs.writeFileSync(dlqFile, "");
  for (const line of lines) {
    if (!line) continue;
    const entry = JSON.parse(line);
    await handler(entry);
  }
}