c50843933f
New service: real-time market data collection with unified event model. Architecture: - Domain events: TradeEvent, QuoteEvent, BookL2Event, HeartbeatEvent - Provider interface: MarketDataProvider ABC with connect/subscribe/stream/close - Async EventBus with fan-out to multiple consumers Providers: - BinanceProvider: public WebSocket (trades + bookTicker), no API key needed, auto-reconnect with exponential backoff, heartbeat timeout detection - AlpacaProvider: IEX real-time data + paper trading auth, dry-run mode when no keys configured (heartbeats only) Consumers: - StorageConsumer: SQLite (via SQLAlchemy async) + JSONL append-only log - MetricsConsumer: Prometheus counters, latency histograms, events/sec gauge - PrintConsumer: sampled structured logging (1/100 events) CLI: python -m app run --provider binance --symbols BTCUSDT,ETHUSDT HTTP: /health, /metrics (Prometheus), /latest?symbol=XXX Tests: 19/19 passed (Binance parse, Alpaca parse, bus smoke tests) Config: pydantic-settings + .env, all secrets via environment variables. Co-authored-by: Cursor <cursoragent@cursor.com>
60 lines
1.8 KiB
Python
60 lines
1.8 KiB
Python
"""
|
|
PrintConsumer: structured debug logging (sampled).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from app.config import settings
|
|
from app.domain.events import Event, EventType
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class PrintConsumer:
|
|
"""
|
|
Logs 1 out of every N events for debugging.
|
|
Always logs heartbeats and first event per symbol.
|
|
"""
|
|
|
|
def __init__(self, sample_rate: int | None = None) -> None:
|
|
self._sample_rate = sample_rate or settings.log_sample_rate
|
|
self._count = 0
|
|
self._seen_symbols: set[str] = set()
|
|
|
|
async def handle(self, event: Event) -> None:
|
|
self._count += 1
|
|
symbol = getattr(event, "symbol", None)
|
|
|
|
# Always log heartbeats
|
|
if event.event_type == EventType.HEARTBEAT:
|
|
logger.info(
|
|
"event.heartbeat",
|
|
extra={"provider": event.provider},
|
|
)
|
|
return
|
|
|
|
# Always log first event for a new symbol
|
|
force_log = False
|
|
if symbol and symbol not in self._seen_symbols:
|
|
self._seen_symbols.add(symbol)
|
|
force_log = True
|
|
|
|
# Sample
|
|
if force_log or self._count % self._sample_rate == 0:
|
|
extra = {
|
|
"provider": event.provider,
|
|
"type": event.event_type.value,
|
|
"symbol": symbol or "?",
|
|
"n": self._count,
|
|
}
|
|
|
|
if event.event_type == EventType.TRADE:
|
|
extra["price"] = getattr(event, "price", None)
|
|
extra["size"] = getattr(event, "size", None)
|
|
elif event.event_type == EventType.QUOTE:
|
|
extra["bid"] = getattr(event, "bid", None)
|
|
extra["ask"] = getattr(event, "ask", None)
|
|
|
|
logger.info("event.sample", extra=extra)
|