09dee24342
Producer (market-data-service):
- Backpressure: smart drop policy (heartbeats→quotes→trades preserved)
- Heartbeat monitor: synthetic HeartbeatEvent on provider silence
- Graceful shutdown: WS→bus→storage→DB engine cleanup sequence
- Bybit V5 public WS provider (backup for Binance, no API key needed)
- FailoverManager: health-based provider switching with recovery
- NATS output adapter: md.events.{type}.{symbol} for SenpAI
- /bus-stats endpoint for backpressure monitoring
- Dockerfile + docker-compose.node1.yml integration
- 36 tests (parsing + bus + failover), requirements.lock
Consumer (senpai-md-consumer):
- NATSConsumer: subscribe md.events.>, queue group senpai-md, backpressure
- State store: LatestState + RollingWindow (deque, 60s)
- Feature engine: 11 features (mid, spread, VWAP, return, vol, latency)
- Rule-based signals: long/short on return+volume+spread conditions
- Publisher: rate-limited features + signals + alerts to NATS
- HTTP API: /health, /metrics, /state/latest, /features/latest, /stats
- 10 Prometheus metrics
- Dockerfile + docker-compose.senpai.yml
- 41 tests (parsing + state + features + rate-limit), requirements.lock
CI: ruff + pytest + smoke import for both services
Tests: 77 total passed, lint clean
Co-authored-by: Cursor <cursoragent@cursor.com>
140 lines
3.4 KiB
Python
140 lines
3.4 KiB
Python
"""
|
|
Domain models — mirrors market-data-service event contracts.
|
|
|
|
Tolerant parsing: unknown fields ignored, partial data accepted.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from enum import Enum
|
|
from typing import Optional
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class EventType(str, Enum):
|
|
TRADE = "trade"
|
|
QUOTE = "quote"
|
|
BOOK_L2 = "book_l2"
|
|
HEARTBEAT = "heartbeat"
|
|
|
|
|
|
def _utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _mono_ns() -> int:
|
|
return time.monotonic_ns()
|
|
|
|
|
|
class BaseEvent(BaseModel, extra="ignore"):
|
|
"""Common fields — extra fields silently ignored."""
|
|
|
|
event_type: EventType
|
|
provider: str
|
|
ts_recv: datetime = Field(default_factory=_utc_now)
|
|
ts_recv_mono_ns: int = Field(default_factory=_mono_ns)
|
|
|
|
|
|
class TradeEvent(BaseEvent):
|
|
event_type: EventType = EventType.TRADE
|
|
symbol: str
|
|
price: float
|
|
size: float
|
|
ts_exchange: Optional[datetime] = None
|
|
side: Optional[str] = None
|
|
trade_id: Optional[str] = None
|
|
|
|
|
|
class QuoteEvent(BaseEvent):
|
|
event_type: EventType = EventType.QUOTE
|
|
symbol: str
|
|
bid: float
|
|
ask: float
|
|
bid_size: float
|
|
ask_size: float
|
|
ts_exchange: Optional[datetime] = None
|
|
|
|
|
|
class BookLevel(BaseModel, extra="ignore"):
|
|
price: float
|
|
size: float
|
|
|
|
|
|
class BookL2Event(BaseEvent):
|
|
event_type: EventType = EventType.BOOK_L2
|
|
symbol: str
|
|
bids: list[BookLevel] = Field(default_factory=list)
|
|
asks: list[BookLevel] = Field(default_factory=list)
|
|
ts_exchange: Optional[datetime] = None
|
|
|
|
|
|
class HeartbeatEvent(BaseEvent):
|
|
event_type: EventType = EventType.HEARTBEAT
|
|
|
|
|
|
# Union for parsing
|
|
Event = TradeEvent | QuoteEvent | BookL2Event | HeartbeatEvent
|
|
|
|
|
|
# ── Output models ──────────────────────────────────────────────────────
|
|
|
|
|
|
class FeatureSnapshot(BaseModel):
|
|
"""Published to senpai.features.{symbol}."""
|
|
|
|
symbol: str
|
|
ts: datetime = Field(default_factory=_utc_now)
|
|
features: dict[str, float | None]
|
|
|
|
|
|
class TradeSignal(BaseModel):
|
|
"""Published to senpai.signals.{symbol}."""
|
|
|
|
symbol: str
|
|
ts: datetime = Field(default_factory=_utc_now)
|
|
direction: str # "long" | "short"
|
|
confidence: float = 0.0 # 0..1
|
|
reason: str = ""
|
|
features: dict[str, float | None] = Field(default_factory=dict)
|
|
|
|
|
|
class AlertEvent(BaseModel):
|
|
"""Published to senpai.alerts."""
|
|
|
|
ts: datetime = Field(default_factory=_utc_now)
|
|
level: str = "warning" # "warning" | "critical"
|
|
alert_type: str # "latency" | "gap" | "backpressure"
|
|
message: str
|
|
details: dict = Field(default_factory=dict)
|
|
|
|
|
|
# ── Parsing helper ─────────────────────────────────────────────────────
|
|
|
|
_EVENT_MAP: dict[str, type[BaseEvent]] = {
|
|
"trade": TradeEvent,
|
|
"quote": QuoteEvent,
|
|
"book_l2": BookL2Event,
|
|
"heartbeat": HeartbeatEvent,
|
|
}
|
|
|
|
|
|
def parse_event(data: dict) -> Event | None:
|
|
"""
|
|
Parse a dict (from JSON) into the appropriate Event model.
|
|
Returns None if event_type is unknown or data is invalid.
|
|
"""
|
|
event_type = data.get("event_type")
|
|
if not event_type:
|
|
return None
|
|
|
|
cls = _EVENT_MAP.get(event_type)
|
|
if cls is None:
|
|
return None
|
|
|
|
try:
|
|
return cls.model_validate(data)
|
|
except Exception:
|
|
return None
|