Files
microdao-daarion/services/market-data-service/app/providers/bybit.py
Apple 09dee24342 feat: MD pipeline — market-data-service hardening + SenpAI NATS consumer
Producer (market-data-service):
- Backpressure: smart drop policy (heartbeats→quotes→trades preserved)
- Heartbeat monitor: synthetic HeartbeatEvent on provider silence
- Graceful shutdown: WS→bus→storage→DB engine cleanup sequence
- Bybit V5 public WS provider (backup for Binance, no API key needed)
- FailoverManager: health-based provider switching with recovery
- NATS output adapter: md.events.{type}.{symbol} for SenpAI
- /bus-stats endpoint for backpressure monitoring
- Dockerfile + docker-compose.node1.yml integration
- 36 tests (parsing + bus + failover), requirements.lock

Consumer (senpai-md-consumer):
- NATSConsumer: subscribe md.events.>, queue group senpai-md, backpressure
- State store: LatestState + RollingWindow (deque, 60s)
- Feature engine: 11 features (mid, spread, VWAP, return, vol, latency)
- Rule-based signals: long/short on return+volume+spread conditions
- Publisher: rate-limited features + signals + alerts to NATS
- HTTP API: /health, /metrics, /state/latest, /features/latest, /stats
- 10 Prometheus metrics
- Dockerfile + docker-compose.senpai.yml
- 41 tests (parsing + state + features + rate-limit), requirements.lock

CI: ruff + pytest + smoke import for both services
Tests: 77 total passed, lint clean
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-09 11:46:15 -08:00

240 lines
7.5 KiB
Python

"""
Bybit V5 public WebSocket provider — backup for Binance.
Streams:
- publicTrade.{symbol} → TradeEvent
- tickers.{symbol} → QuoteEvent (best bid/ask from tickers)
Docs: https://bybit-exchange.github.io/docs/v5/ws/connect
No API key needed for public market data.
"""
from __future__ import annotations
import asyncio
import json
import logging
from datetime import datetime, timezone
from typing import AsyncIterator
import websockets
from websockets.exceptions import ConnectionClosed
from app.config import settings
from app.domain.events import (
Event,
QuoteEvent,
TradeEvent,
)
from app.providers import MarketDataProvider
logger = logging.getLogger(__name__)
def _ms_to_dt(ms: int | float | str | None) -> datetime | None:
"""Convert millisecond epoch to UTC datetime."""
if ms is None:
return None
try:
return datetime.fromtimestamp(int(ms) / 1000.0, tz=timezone.utc)
except (ValueError, TypeError, OSError):
return None
class BybitProvider(MarketDataProvider):
"""
Bybit V5 public WebSocket (spot market).
Connects to the spot public channel and subscribes to
publicTrade + tickers for each symbol.
"""
name = "bybit"
def __init__(self) -> None:
self._ws: websockets.WebSocketClientProtocol | None = None
self._symbols: list[str] = []
self._connected = False
self._reconnect_count = 0
self._base_url = settings.bybit_ws_url
async def connect(self) -> None:
"""Establish WebSocket connection."""
logger.info("bybit.connecting", extra={"url": self._base_url})
self._ws = await websockets.connect(
self._base_url,
ping_interval=20,
ping_timeout=10,
close_timeout=5,
)
self._connected = True
logger.info("bybit.connected")
async def subscribe(self, symbols: list[str]) -> None:
"""Subscribe to publicTrade + tickers for each symbol."""
if not self._ws:
raise RuntimeError("Not connected. Call connect() first.")
self._symbols = [s.upper() for s in symbols]
args = []
for sym in self._symbols:
args.append(f"publicTrade.{sym}")
args.append(f"tickers.{sym}")
subscribe_msg = {
"op": "subscribe",
"args": args,
}
await self._ws.send(json.dumps(subscribe_msg))
logger.info(
"bybit.subscribed",
extra={"symbols": self._symbols, "channels": len(args)},
)
async def stream(self) -> AsyncIterator[Event]:
"""Yield domain events. Handles reconnect automatically."""
backoff = settings.reconnect_base_delay
while True:
try:
if not self._connected or not self._ws:
await self._reconnect(backoff)
try:
raw = await asyncio.wait_for(
self._ws.recv(), # type: ignore
timeout=settings.heartbeat_timeout,
)
except asyncio.TimeoutError:
logger.warning(
"bybit.heartbeat_timeout",
extra={"timeout": settings.heartbeat_timeout},
)
self._connected = False
continue
# Reset backoff on successful message
backoff = settings.reconnect_base_delay
data = json.loads(raw)
# Handle pong (Bybit sends {"op":"pong",...})
if data.get("op") in ("pong", "subscribe"):
if data.get("success") is False:
logger.warning("bybit.subscribe_failed", extra={"msg": data})
continue
event = self._parse(data)
if event:
yield event
except ConnectionClosed as e:
logger.warning(
"bybit.connection_closed",
extra={"code": e.code, "reason": str(e.reason)},
)
self._connected = False
backoff = min(backoff * 2, settings.reconnect_max_delay)
except Exception as e:
logger.error("bybit.stream_error", extra={"error": str(e)})
self._connected = False
backoff = min(backoff * 2, settings.reconnect_max_delay)
async def _reconnect(self, delay: float) -> None:
"""Reconnect with delay, then resubscribe."""
self._reconnect_count += 1
logger.info(
"bybit.reconnecting",
extra={"delay": delay, "attempt": self._reconnect_count},
)
await asyncio.sleep(delay)
try:
if self._ws:
await self._ws.close()
except Exception:
pass
await self.connect()
if self._symbols:
await self.subscribe(self._symbols)
def _parse(self, data: dict) -> Event | None:
"""Parse raw Bybit V5 message into domain events."""
topic = data.get("topic", "")
event_data = data.get("data")
if not topic or event_data is None:
return None
if topic.startswith("publicTrade."):
return self._parse_trades(event_data)
elif topic.startswith("tickers."):
return self._parse_ticker(event_data)
return None
def _parse_trades(self, data: list | dict) -> Event | None:
"""
Bybit publicTrade payload (V5):
{"data": [{"s":"BTCUSDT","S":"Buy","v":"0.001","p":"70000.5","T":1672515782136,"i":"..."}]}
We take the last trade in the batch.
"""
if isinstance(data, list):
if not data:
return None
trade = data[-1] # latest in batch
else:
trade = data
return TradeEvent(
provider=self.name,
symbol=trade.get("s", "").upper(),
price=float(trade.get("p", 0)),
size=float(trade.get("v", 0)),
ts_exchange=_ms_to_dt(trade.get("T")),
side=trade.get("S", "").lower() if trade.get("S") else None,
trade_id=str(trade.get("i", "")),
)
def _parse_ticker(self, data: dict) -> QuoteEvent | None:
"""
Bybit tickers (V5 spot):
{"data": {"symbol":"BTCUSDT","bid1Price":"70000.5","bid1Size":"1.5",
"ask1Price":"70001.0","ask1Size":"2.0",...}}
"""
if isinstance(data, list):
data = data[0] if data else {}
bid = data.get("bid1Price") or data.get("bidPrice")
ask = data.get("ask1Price") or data.get("askPrice")
bid_size = data.get("bid1Size") or data.get("bidSize")
ask_size = data.get("ask1Size") or data.get("askSize")
if not bid or not ask:
return None
return QuoteEvent(
provider=self.name,
symbol=data.get("symbol", "").upper(),
bid=float(bid),
ask=float(ask),
bid_size=float(bid_size or 0),
ask_size=float(ask_size or 0),
ts_exchange=_ms_to_dt(data.get("ts")),
)
async def close(self) -> None:
"""Close the WebSocket connection."""
self._connected = False
if self._ws:
try:
await self._ws.close()
except Exception:
pass
logger.info(
"bybit.closed",
extra={"reconnect_count": self._reconnect_count},
)