Producer (market-data-service):
- Backpressure: smart drop policy (heartbeats→quotes→trades preserved)
- Heartbeat monitor: synthetic HeartbeatEvent on provider silence
- Graceful shutdown: WS→bus→storage→DB engine cleanup sequence
- Bybit V5 public WS provider (backup for Binance, no API key needed)
- FailoverManager: health-based provider switching with recovery
- NATS output adapter: md.events.{type}.{symbol} for SenpAI
- /bus-stats endpoint for backpressure monitoring
- Dockerfile + docker-compose.node1.yml integration
- 36 tests (parsing + bus + failover), requirements.lock
Consumer (senpai-md-consumer):
- NATSConsumer: subscribe md.events.>, queue group senpai-md, backpressure
- State store: LatestState + RollingWindow (deque, 60s)
- Feature engine: 11 features (mid, spread, VWAP, return, vol, latency)
- Rule-based signals: long/short on return+volume+spread conditions
- Publisher: rate-limited features + signals + alerts to NATS
- HTTP API: /health, /metrics, /state/latest, /features/latest, /stats
- 10 Prometheus metrics
- Dockerfile + docker-compose.senpai.yml
- 41 tests (parsing + state + features + rate-limit), requirements.lock
CI: ruff + pytest + smoke import for both services
Tests: 77 total passed, lint clean
Co-authored-by: Cursor <cursoragent@cursor.com>
222 lines
6.8 KiB
Python
222 lines
6.8 KiB
Python
"""
|
|
Binance public WebSocket provider.
|
|
|
|
No API key required. Subscribes to:
|
|
- <symbol>@trade → TradeEvent
|
|
- <symbol>@bookTicker → QuoteEvent
|
|
|
|
Auto-reconnect with exponential backoff via tenacity.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import AsyncIterator
|
|
|
|
import websockets
|
|
from websockets.exceptions import ConnectionClosed
|
|
|
|
from app.config import settings
|
|
from app.domain.events import (
|
|
Event,
|
|
QuoteEvent,
|
|
TradeEvent,
|
|
)
|
|
from app.providers import MarketDataProvider
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _ms_to_dt(ms: int | float | None) -> datetime | None:
|
|
"""Convert millisecond epoch to UTC datetime."""
|
|
if ms is None:
|
|
return None
|
|
return datetime.fromtimestamp(ms / 1000.0, tz=timezone.utc)
|
|
|
|
|
|
class BinanceProvider(MarketDataProvider):
|
|
"""
|
|
Binance public WebSocket streams.
|
|
|
|
Connects to the combined stream endpoint and subscribes to
|
|
trade + bookTicker channels for each symbol.
|
|
"""
|
|
|
|
name = "binance"
|
|
|
|
def __init__(self) -> None:
|
|
self._ws: websockets.WebSocketClientProtocol | None = None
|
|
self._symbols: list[str] = []
|
|
self._connected = False
|
|
self._reconnect_count = 0
|
|
self._base_url = settings.binance_ws_url
|
|
|
|
async def connect(self) -> None:
|
|
"""Establish WebSocket connection."""
|
|
logger.info("binance.connecting", extra={"url": self._base_url})
|
|
self._ws = await websockets.connect(
|
|
self._base_url,
|
|
ping_interval=20,
|
|
ping_timeout=10,
|
|
close_timeout=5,
|
|
)
|
|
self._connected = True
|
|
logger.info("binance.connected")
|
|
|
|
async def subscribe(self, symbols: list[str]) -> None:
|
|
"""Subscribe to trade + bookTicker for each symbol."""
|
|
if not self._ws:
|
|
raise RuntimeError("Not connected. Call connect() first.")
|
|
|
|
self._symbols = [s.lower() for s in symbols]
|
|
streams = []
|
|
for sym in self._symbols:
|
|
streams.append(f"{sym}@trade")
|
|
streams.append(f"{sym}@bookTicker")
|
|
|
|
subscribe_msg = {
|
|
"method": "SUBSCRIBE",
|
|
"params": streams,
|
|
"id": 1,
|
|
}
|
|
await self._ws.send(json.dumps(subscribe_msg))
|
|
logger.info(
|
|
"binance.subscribed",
|
|
extra={"symbols": self._symbols, "streams": len(streams)},
|
|
)
|
|
|
|
async def stream(self) -> AsyncIterator[Event]:
|
|
"""
|
|
Yield domain events. Handles reconnect automatically.
|
|
"""
|
|
backoff = settings.reconnect_base_delay
|
|
|
|
while True:
|
|
try:
|
|
if not self._connected or not self._ws:
|
|
await self._reconnect(backoff)
|
|
|
|
# Set timeout for heartbeat detection
|
|
try:
|
|
raw = await asyncio.wait_for(
|
|
self._ws.recv(), # type: ignore
|
|
timeout=settings.heartbeat_timeout,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.warning(
|
|
"binance.heartbeat_timeout",
|
|
extra={"timeout": settings.heartbeat_timeout},
|
|
)
|
|
self._connected = False
|
|
continue
|
|
|
|
# Reset backoff on successful message
|
|
backoff = settings.reconnect_base_delay
|
|
|
|
data = json.loads(raw)
|
|
|
|
# Skip subscription confirmations
|
|
if "result" in data and "id" in data:
|
|
continue
|
|
|
|
event = self._parse(data)
|
|
if event:
|
|
yield event
|
|
|
|
except ConnectionClosed as e:
|
|
logger.warning(
|
|
"binance.connection_closed",
|
|
extra={"code": e.code, "reason": str(e.reason)},
|
|
)
|
|
self._connected = False
|
|
backoff = min(backoff * 2, settings.reconnect_max_delay)
|
|
|
|
except Exception as e:
|
|
logger.error("binance.stream_error", extra={"error": str(e)})
|
|
self._connected = False
|
|
backoff = min(backoff * 2, settings.reconnect_max_delay)
|
|
|
|
async def _reconnect(self, delay: float) -> None:
|
|
"""Reconnect with delay, then resubscribe."""
|
|
self._reconnect_count += 1
|
|
logger.info(
|
|
"binance.reconnecting",
|
|
extra={"delay": delay, "attempt": self._reconnect_count},
|
|
)
|
|
await asyncio.sleep(delay)
|
|
|
|
try:
|
|
if self._ws:
|
|
await self._ws.close()
|
|
except Exception:
|
|
pass
|
|
|
|
await self.connect()
|
|
if self._symbols:
|
|
await self.subscribe(self._symbols)
|
|
|
|
def _parse(self, data: dict) -> Event | None:
|
|
"""Parse raw Binance JSON into domain events."""
|
|
event_type = data.get("e")
|
|
|
|
if event_type == "trade":
|
|
return self._parse_trade(data)
|
|
elif event_type == "bookTicker" or ("b" in data and "a" in data and "s" in data and "e" not in data):
|
|
# bookTicker doesn't always have "e" field in combined stream
|
|
return self._parse_book_ticker(data)
|
|
|
|
return None
|
|
|
|
def _parse_trade(self, data: dict) -> TradeEvent:
|
|
"""
|
|
Binance trade payload:
|
|
{
|
|
"e": "trade", "E": 1672515782136, "s": "BNBBTC",
|
|
"t": 12345, "p": "0.001", "q": "100",
|
|
"T": 1672515782136, "m": true
|
|
}
|
|
"""
|
|
return TradeEvent(
|
|
provider=self.name,
|
|
symbol=data.get("s", "").upper(),
|
|
price=float(data.get("p", 0)),
|
|
size=float(data.get("q", 0)),
|
|
ts_exchange=_ms_to_dt(data.get("T")),
|
|
side="sell" if data.get("m") else "buy", # m=True → buyer is maker → trade is a sell
|
|
trade_id=str(data.get("t", "")),
|
|
)
|
|
|
|
def _parse_book_ticker(self, data: dict) -> QuoteEvent:
|
|
"""
|
|
Binance bookTicker payload:
|
|
{
|
|
"u": 400900217, "s": "BNBUSDT",
|
|
"b": "25.35190000", "B": "31.21000000",
|
|
"a": "25.36520000", "A": "40.66000000"
|
|
}
|
|
"""
|
|
return QuoteEvent(
|
|
provider=self.name,
|
|
symbol=data.get("s", "").upper(),
|
|
bid=float(data.get("b", 0)),
|
|
ask=float(data.get("a", 0)),
|
|
bid_size=float(data.get("B", 0)),
|
|
ask_size=float(data.get("A", 0)),
|
|
ts_exchange=_ms_to_dt(data.get("E")),
|
|
)
|
|
|
|
async def close(self) -> None:
|
|
"""Close the WebSocket connection."""
|
|
self._connected = False
|
|
if self._ws:
|
|
try:
|
|
await self._ws.close()
|
|
except Exception:
|
|
pass
|
|
logger.info(
|
|
"binance.closed",
|
|
extra={"reconnect_count": self._reconnect_count},
|
|
)
|