feat: Upgrade Global Presence to SSE architecture

- matrix-presence-aggregator v2 with SSE endpoint
- Created @presence_daemon Matrix user
- SSE proxy in Next.js /api/presence/stream
- Updated frontend to use SSE instead of WebSocket
- Real-time city online count and room presence

services/matrix-presence-aggregator/app/__init__.py

@@ -0,0 +1,2 @@
+# Matrix Presence Aggregator
+

services/matrix-presence-aggregator/app/aggregator.py

@@ -0,0 +1,154 @@
+"""Presence aggregation logic with caching and broadcasting"""
+import asyncio
+from datetime import datetime, timezone
+from typing import List, Optional
+import logging
+
+from .models import PresenceSnapshot, RoomPresence, CityPresence
+from .matrix_client import MatrixClient
+from .rooms_source import RoomsSource
+
+logger = logging.getLogger(__name__)
+
+
+class PresenceAggregator:
+    """
+    Aggregates presence data from Matrix and broadcasts to subscribers.
+    
+    - Periodically polls Matrix for room members and presence
+    - Caches the latest snapshot
+    - Broadcasts updates to SSE subscribers
+    """
+    
+    def __init__(
+        self,
+        matrix_client: MatrixClient,
+        rooms_source: RoomsSource,
+        poll_interval_seconds: int = 5,
+    ):
+        self.matrix_client = matrix_client
+        self.rooms_source = rooms_source
+        self.poll_interval_seconds = poll_interval_seconds
+
+        self._snapshot: Optional[PresenceSnapshot] = None
+        self._subscribers: List[asyncio.Queue] = []
+        self._running = False
+
+    def get_snapshot(self) -> Optional[PresenceSnapshot]:
+        """Get the latest cached snapshot"""
+        return self._snapshot
+
+    def register_subscriber(self) -> asyncio.Queue:
+        """Register a new SSE subscriber"""
+        q: asyncio.Queue = asyncio.Queue()
+        self._subscribers.append(q)
+        logger.info(f"Subscriber registered. Total: {len(self._subscribers)}")
+        return q
+
+    def unregister_subscriber(self, q: asyncio.Queue):
+        """Unregister an SSE subscriber"""
+        if q in self._subscribers:
+            self._subscribers.remove(q)
+            logger.info(f"Subscriber unregistered. Total: {len(self._subscribers)}")
+
+    async def _broadcast(self, snapshot: PresenceSnapshot):
+        """Broadcast snapshot to all subscribers"""
+        for q in list(self._subscribers):
+            try:
+                # Don't block if queue is full
+                if q.qsize() < 10:
+                    await q.put(snapshot)
+            except asyncio.CancelledError:
+                pass
+
+    async def _compute_snapshot(self) -> PresenceSnapshot:
+        """Compute a new presence snapshot from Matrix"""
+        rooms = self.rooms_source.get_rooms()
+        
+        if not rooms:
+            logger.warning("No rooms with matrix_room_id found")
+
+        room_presences: List[RoomPresence] = []
+        city_online_total = 0
+        rooms_online = 0
+
+        for r in rooms:
+            matrix_room_id = r["matrix_room_id"]
+            
+            try:
+                # Get room members
+                members = await self.matrix_client.get_room_members(matrix_room_id)
+                
+                # Get presence for each member
+                online_count = 0
+                for member in members:
+                    user_id = member.get("user_id")
+                    if not user_id:
+                        continue
+                    
+                    presence = await self.matrix_client.get_presence(user_id)
+                    if presence in ("online", "unavailable"):
+                        online_count += 1
+
+                # Get typing (currently returns empty, needs sync loop)
+                typing_users = await self.matrix_client.get_room_typing(matrix_room_id)
+                typing_count = len(typing_users)
+
+                if online_count > 0:
+                    rooms_online += 1
+
+                city_online_total += online_count
+
+                room_presences.append(
+                    RoomPresence(
+                        room_id=r["room_id"],
+                        matrix_room_id=matrix_room_id,
+                        online=online_count,
+                        typing=typing_count,
+                    )
+                )
+                
+            except Exception as e:
+                logger.error(f"Error processing room {r['room_id']}: {e}")
+                # Add room with 0 online
+                room_presences.append(
+                    RoomPresence(
+                        room_id=r["room_id"],
+                        matrix_room_id=matrix_room_id,
+                        online=0,
+                        typing=0,
+                    )
+                )
+
+        snapshot = PresenceSnapshot(
+            timestamp=datetime.now(timezone.utc),
+            city=CityPresence(
+                online_total=city_online_total,
+                rooms_online=rooms_online,
+            ),
+            rooms=room_presences,
+        )
+        
+        logger.info(f"Computed snapshot: {city_online_total} online in {rooms_online} rooms")
+        return snapshot
+
+    async def run_forever(self):
+        """Main loop - continuously compute and broadcast snapshots"""
+        self._running = True
+        logger.info(f"Starting presence aggregator (poll interval: {self.poll_interval_seconds}s)")
+        
+        while self._running:
+            try:
+                snapshot = await self._compute_snapshot()
+                self._snapshot = snapshot
+                await self._broadcast(snapshot)
+            except Exception as e:
+                logger.error(f"Error in aggregator loop: {e}")
+            
+            await asyncio.sleep(self.poll_interval_seconds)
+
+    def stop(self):
+        """Stop the aggregator loop"""
+        self._running = False
+        logger.info("Stopping presence aggregator")
+

services/matrix-presence-aggregator/app/config.py

@@ -0,0 +1,36 @@
+"""Configuration for Matrix Presence Aggregator"""
+from pydantic import BaseModel
+import os
+
+
+class Settings(BaseModel):
+    matrix_base_url: str
+    matrix_access_token: str
+    matrix_homeserver_domain: str = "daarion.space"
+    poll_interval_seconds: int = 5
+
+    rooms_source: str = "database"  # "database" | "static"
+    db_dsn: str | None = None
+    rooms_config_path: str | None = None
+
+    http_host: str = "0.0.0.0"
+    http_port: int = 8085
+    
+    # Filter out presence daemon from member lists
+    presence_daemon_user: str = "@presence_daemon:daarion.space"
+
+
+def load_settings() -> Settings:
+    return Settings(
+        matrix_base_url=os.getenv("MATRIX_BASE_URL", "https://app.daarion.space"),
+        matrix_access_token=os.getenv("MATRIX_ACCESS_TOKEN", ""),
+        matrix_homeserver_domain=os.getenv("MATRIX_HOMESERVER_DOMAIN", "daarion.space"),
+        poll_interval_seconds=int(os.getenv("POLL_INTERVAL_SECONDS", "5")),
+        rooms_source=os.getenv("ROOMS_SOURCE", "database"),
+        db_dsn=os.getenv("DB_DSN"),
+        rooms_config_path=os.getenv("ROOMS_CONFIG"),
+        http_host=os.getenv("PRESENCE_HTTP_HOST", "0.0.0.0"),
+        http_port=int(os.getenv("PRESENCE_HTTP_PORT", "8085")),
+        presence_daemon_user=os.getenv("PRESENCE_DAEMON_USER", "@presence_daemon:daarion.space"),
+    )
+

services/matrix-presence-aggregator/app/main.py

@@ -0,0 +1,159 @@
+"""
+Matrix Presence Aggregator - FastAPI Application
+
+Provides REST and SSE endpoints for real-time presence data.
+"""
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.middleware.cors import CORSMiddleware
+import asyncio
+import uvicorn
+import logging
+
+from .config import load_settings
+from .matrix_client import MatrixClient
+from .rooms_source import RoomsSource, StaticRoomsSource
+from .aggregator import PresenceAggregator
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+settings = load_settings()
+
+app = FastAPI(
+    title="Matrix Presence Aggregator",
+    description="Real-time presence aggregation for DAARION City",
+    version="2.0.0"
+)
+
+# CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Initialize components
+matrix_client = MatrixClient(
+    base_url=settings.matrix_base_url,
+    access_token=settings.matrix_access_token,
+    daemon_user=settings.presence_daemon_user,
+)
+
+# Choose rooms source
+if settings.rooms_source == "database" and settings.db_dsn:
+    rooms_source = RoomsSource(db_dsn=settings.db_dsn)
+    logger.info(f"Using database rooms source: {settings.db_dsn[:30]}...")
+elif settings.rooms_source == "static" and settings.rooms_config_path:
+    rooms_source = StaticRoomsSource(config_path=settings.rooms_config_path)
+    logger.info(f"Using static rooms source: {settings.rooms_config_path}")
+else:
+    # Fallback to database with default DSN
+    rooms_source = RoomsSource(db_dsn=settings.db_dsn or "postgresql://postgres:postgres@localhost:5432/postgres")
+    logger.warning("No rooms source configured, using default database")
+
+aggregator = PresenceAggregator(
+    matrix_client=matrix_client,
+    rooms_source=rooms_source,
+    poll_interval_seconds=settings.poll_interval_seconds,
+)
+
+
+@app.on_event("startup")
+async def startup_event():
+    logger.info("Starting Matrix Presence Aggregator...")
+    asyncio.create_task(aggregator.run_forever())
+    logger.info("Aggregator task started")
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    logger.info("Shutting down...")
+    aggregator.stop()
+    await matrix_client.close()
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    snapshot = aggregator.get_snapshot()
+    return {
+        "status": "healthy",
+        "service": "matrix-presence-aggregator",
+        "has_snapshot": snapshot is not None,
+        "subscribers": len(aggregator._subscribers),
+    }
+
+
+@app.get("/presence/summary")
+async def get_presence_summary():
+    """
+    Get current presence snapshot.
+    
+    Returns aggregated presence data for all rooms.
+    """
+    snapshot = aggregator.get_snapshot()
+    if snapshot is None:
+        return JSONResponse(
+            content={"status": "initializing", "message": "Waiting for first poll"},
+            status_code=503,
+        )
+    return snapshot.model_dump()
+
+
+@app.get("/presence/stream")
+async def presence_stream(request: Request):
+    """
+    SSE stream of presence updates.
+    
+    Clients receive real-time updates whenever presence changes.
+    """
+    async def event_generator():
+        q = aggregator.register_subscriber()
+        
+        # Send initial snapshot immediately
+        initial = aggregator.get_snapshot()
+        if initial is not None:
+            yield f"data: {initial.model_dump_json()}\n\n"
+
+        try:
+            while True:
+                if await request.is_disconnected():
+                    break
+
+                try:
+                    snapshot = await asyncio.wait_for(q.get(), timeout=15.0)
+                    yield f"data: {snapshot.model_dump_json()}\n\n"
+                except asyncio.TimeoutError:
+                    # Keep connection alive
+                    yield ": keep-alive\n\n"
+                    continue
+
+        finally:
+            aggregator.unregister_subscriber(q)
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",  # Disable nginx buffering
+        }
+    )
+
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "app.main:app",
+        host=settings.http_host,
+        port=settings.http_port,
+        reload=True,
+    )
+

services/matrix-presence-aggregator/app/matrix_client.py

@@ -0,0 +1,94 @@
+"""Matrix API client for presence aggregation"""
+import httpx
+from typing import List, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class MatrixClient:
+    """Simplified Matrix client for reading members, presence, and typing"""
+    
+    def __init__(self, base_url: str, access_token: str, daemon_user: str = ""):
+        self.base_url = base_url.rstrip("/")
+        self.access_token = access_token
+        self.daemon_user = daemon_user  # Filter this user from lists
+        self._client = httpx.AsyncClient(
+            base_url=self.base_url,
+            headers={"Authorization": f"Bearer {self.access_token}"},
+            timeout=30.0
+        )
+
+    async def get_room_members(self, room_id: str) -> List[dict]:
+        """Get all members of a room"""
+        try:
+            # Use joined_members for efficiency
+            res = await self._client.get(
+                f"/_matrix/client/v3/rooms/{room_id}/joined_members"
+            )
+            res.raise_for_status()
+            data = res.json()
+            
+            # joined_members returns: {"joined": {"@user:domain": {...}}}
+            joined = data.get("joined", {})
+            members = []
+            for user_id, info in joined.items():
+                # Filter out presence daemon
+                if user_id == self.daemon_user:
+                    continue
+                members.append({
+                    "user_id": user_id,
+                    "display_name": info.get("display_name"),
+                    "avatar_url": info.get("avatar_url"),
+                })
+            return members
+        except httpx.HTTPError as e:
+            logger.error(f"Failed to get room members for {room_id}: {e}")
+            return []
+
+    async def get_room_typing(self, room_id: str) -> List[str]:
+        """Get list of currently typing users in a room"""
+        # Note: Matrix doesn't have a direct API for this
+        # Typing info comes from /sync, which we'd need to run continuously
+        # For now, return empty - we'll get typing from sync loop later
+        return []
+
+    async def get_presence(self, user_id: str) -> str:
+        """Get presence status for a user"""
+        try:
+            res = await self._client.get(
+                f"/_matrix/client/v3/presence/{user_id}/status"
+            )
+            if res.status_code != 200:
+                return "offline"
+            data = res.json()
+            return data.get("presence", "offline")
+        except httpx.HTTPError:
+            return "offline"
+
+    async def get_presence_batch(self, user_ids: List[str]) -> dict:
+        """Get presence for multiple users (with caching)"""
+        # For efficiency, we could batch these or use sync
+        # For now, simple sequential calls with error handling
+        result = {}
+        for user_id in user_ids:
+            result[user_id] = await self.get_presence(user_id)
+        return result
+
+    async def join_room(self, room_id_or_alias: str) -> Optional[str]:
+        """Join a room and return the room_id"""
+        try:
+            res = await self._client.post(
+                f"/_matrix/client/v3/join/{room_id_or_alias}",
+                json={}
+            )
+            res.raise_for_status()
+            data = res.json()
+            return data.get("room_id")
+        except httpx.HTTPError as e:
+            logger.error(f"Failed to join room {room_id_or_alias}: {e}")
+            return None
+
+    async def close(self):
+        await self._client.aclose()
+

services/matrix-presence-aggregator/app/models.py

@@ -0,0 +1,24 @@
+"""Data models for Presence Aggregator"""
+from pydantic import BaseModel
+from typing import List
+from datetime import datetime
+
+
+class RoomPresence(BaseModel):
+    room_id: str          # internal room id from DB
+    matrix_room_id: str   # Matrix room ID (!xxx:domain)
+    online: int
+    typing: int
+
+
+class CityPresence(BaseModel):
+    online_total: int
+    rooms_online: int
+
+
+class PresenceSnapshot(BaseModel):
+    type: str = "presence_update"
+    timestamp: datetime
+    city: CityPresence
+    rooms: List[RoomPresence]
+

services/matrix-presence-aggregator/app/rooms_source.py

@@ -0,0 +1,69 @@
+"""Room source - reads rooms from database or static config"""
+from sqlalchemy import create_engine, text
+from typing import List, Dict
+import logging
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class RoomsSource:
+    """Reads room list from PostgreSQL database"""
+    
+    def __init__(self, db_dsn: str):
+        self.engine = create_engine(db_dsn)
+
+    def get_rooms(self) -> List[Dict]:
+        """
+        Get all rooms with matrix_room_id set.
+        
+        Expected table structure:
+        - id (text)
+        - slug (text) 
+        - name (text)
+        - matrix_room_id (text, nullable)
+        """
+        query = text(
+            """
+            SELECT id, slug, name, matrix_room_id
+            FROM city_rooms
+            WHERE matrix_room_id IS NOT NULL
+            """
+        )
+        try:
+            with self.engine.connect() as conn:
+                rows = conn.execute(query).mappings().all()
+
+            return [
+                {
+                    "room_id": str(r["id"]),
+                    "slug": r["slug"],
+                    "title": r["name"],
+                    "matrix_room_id": r["matrix_room_id"],
+                }
+                for r in rows
+            ]
+        except Exception as e:
+            logger.error(f"Failed to get rooms from database: {e}")
+            return []
+
+
+class StaticRoomsSource:
+    """Reads room list from YAML config file"""
+    
+    def __init__(self, config_path: str):
+        self.config_path = config_path
+        self._rooms = self._load_config()
+
+    def _load_config(self) -> List[Dict]:
+        try:
+            with open(self.config_path, 'r') as f:
+                data = yaml.safe_load(f)
+                return data.get('rooms', [])
+        except Exception as e:
+            logger.error(f"Failed to load rooms config: {e}")
+            return []
+
+    def get_rooms(self) -> List[Dict]:
+        return self._rooms
+