claude-mpm 4.1.4__py3-none-any.whl → 4.1.6__py3-none-any.whl

claude_mpm/services/socketio/server/connection_manager.py

@@ -0,0 +1,516 @@
+"""
+Enhanced Connection Manager for SocketIO Server.
+
+WHY: This module provides robust connection management with state tracking,
+health monitoring, event buffering for disconnected clients, and automatic
+recovery from connection failures.
+
+DESIGN DECISION: Centralized connection management ensures consistent handling
+of client states, proper event delivery, and automatic recovery mechanisms.
+"""
+
+import asyncio
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Deque, Dict, List, Optional
+from uuid import uuid4
+
+from ....core.logging_config import get_logger
+
+
+class ConnectionState(Enum):
+    """Connection states for tracking client lifecycle."""
+
+    CONNECTING = "connecting"
+    CONNECTED = "connected"
+    DISCONNECTING = "disconnecting"
+    DISCONNECTED = "disconnected"
+    RECONNECTING = "reconnecting"
+    STALE = "stale"  # Connected but not responding
+
+
+@dataclass
+class ConnectionMetrics:
+    """Metrics for a single connection."""
+
+    connect_count: int = 0
+    disconnect_count: int = 0
+    reconnect_count: int = 0
+    events_sent: int = 0
+    events_acked: int = 0
+    events_buffered: int = 0
+    events_dropped: int = 0
+    last_activity: float = field(default_factory=time.time)
+    total_uptime: float = 0.0
+    total_downtime: float = 0.0
+    connection_quality: float = 1.0  # 0-1 quality score
+
+
+@dataclass
+class ClientConnection:
+    """Represents a client connection with full state tracking."""
+
+    sid: str  # Socket ID
+    client_id: str  # Persistent client ID across reconnections
+    state: ConnectionState
+    connected_at: float
+    disconnected_at: Optional[float] = None
+    last_ping: Optional[float] = None
+    last_pong: Optional[float] = None
+    last_event: Optional[float] = None
+    event_buffer: Deque[Dict[str, Any]] = field(
+        default_factory=lambda: deque(maxlen=1000)
+    )
+    event_sequence: int = 0
+    last_acked_sequence: int = 0
+    pending_acks: Dict[int, Dict[str, Any]] = field(default_factory=dict)
+    metrics: ConnectionMetrics = field(default_factory=ConnectionMetrics)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def is_healthy(self, timeout: float = 90.0) -> bool:
+        """Check if connection is healthy based on activity."""
+        if self.state != ConnectionState.CONNECTED:
+            return False
+
+        now = time.time()
+
+        # Check last activity (ping, pong, or event)
+        last_activity = max(
+            self.last_ping or 0,
+            self.last_pong or 0,
+            self.last_event or 0,
+            self.connected_at,
+        )
+
+        return (now - last_activity) < timeout
+
+    def calculate_quality(self) -> float:
+        """Calculate connection quality score (0-1)."""
+        if self.state != ConnectionState.CONNECTED:
+            return 0.0
+
+        # Factors for quality calculation
+        factors = []
+
+        # Reconnection rate (lower is better)
+        if self.metrics.connect_count > 0:
+            reconnect_rate = self.metrics.reconnect_count / self.metrics.connect_count
+            factors.append(1.0 - min(reconnect_rate, 1.0))
+
+        # Event acknowledgment rate
+        if self.metrics.events_sent > 0:
+            ack_rate = self.metrics.events_acked / self.metrics.events_sent
+            factors.append(ack_rate)
+
+        # Uptime ratio
+        total_time = self.metrics.total_uptime + self.metrics.total_downtime
+        if total_time > 0:
+            uptime_ratio = self.metrics.total_uptime / total_time
+            factors.append(uptime_ratio)
+
+        # Recent activity (exponential decay over 5 minutes)
+        now = time.time()
+        time_since_activity = now - self.metrics.last_activity
+        activity_score = max(0, 1.0 - (time_since_activity / 300))
+        factors.append(activity_score)
+
+        # Calculate average quality
+        if factors:
+            quality = sum(factors) / len(factors)
+        else:
+            quality = 1.0 if self.state == ConnectionState.CONNECTED else 0.0
+
+        self.metrics.connection_quality = quality
+        return quality
+
+
+class ConnectionManager:
+    """
+    Enhanced connection manager with robust state tracking and recovery.
+
+    Features:
+    - Persistent client IDs across reconnections
+    - Event buffering for disconnected clients
+    - Sequence numbers for event ordering
+    - Health monitoring with automatic stale detection
+    - Connection quality metrics
+    - Automatic event replay on reconnection
+    """
+
+    def __init__(self, max_buffer_size: int = 1000, event_ttl: int = 300):
+        """
+        Initialize connection manager.
+
+        Args:
+            max_buffer_size: Maximum events to buffer per client
+            event_ttl: Time-to-live for buffered events in seconds
+        """
+        self.logger = get_logger(__name__)
+        self.connections: Dict[str, ClientConnection] = {}
+        self.client_mapping: Dict[str, str] = {}  # client_id -> current sid
+        self.max_buffer_size = max_buffer_size
+        self.event_ttl = event_ttl
+        self.global_sequence = 0
+        self.health_check_interval = 30  # seconds
+        self.stale_timeout = 90  # seconds
+        self.health_task = None
+        self._lock = asyncio.Lock()
+
+    async def register_connection(
+        self, sid: str, client_id: Optional[str] = None
+    ) -> ClientConnection:
+        """
+        Register a new connection or reconnection.
+
+        Args:
+            sid: Socket ID
+            client_id: Optional persistent client ID for reconnection
+
+        Returns:
+            ClientConnection object
+        """
+        async with self._lock:
+            now = time.time()
+
+            # Check if this is a reconnection
+            if client_id and client_id in self.client_mapping:
+                old_sid = self.client_mapping[client_id]
+                if old_sid in self.connections:
+                    old_conn = self.connections[old_sid]
+
+                    # Create new connection with history
+                    conn = ClientConnection(
+                        sid=sid,
+                        client_id=client_id,
+                        state=ConnectionState.CONNECTED,
+                        connected_at=now,
+                        event_buffer=old_conn.event_buffer,
+                        event_sequence=old_conn.event_sequence,
+                        last_acked_sequence=old_conn.last_acked_sequence,
+                        metrics=old_conn.metrics,
+                    )
+
+                    # Update metrics
+                    conn.metrics.reconnect_count += 1
+                    conn.metrics.connect_count += 1
+                    if old_conn.disconnected_at:
+                        conn.metrics.total_downtime += now - old_conn.disconnected_at
+
+                    # Clean up old connection
+                    del self.connections[old_sid]
+
+                    self.logger.info(
+                        f"Client {client_id} reconnected (new sid: {sid}, "
+                        f"buffered events: {len(conn.event_buffer)})"
+                    )
+                else:
+                    # No old connection found, create new
+                    client_id = client_id or str(uuid4())
+                    conn = self._create_new_connection(sid, client_id, now)
+            else:
+                # New client
+                client_id = client_id or str(uuid4())
+                conn = self._create_new_connection(sid, client_id, now)
+
+            # Register connection
+            self.connections[sid] = conn
+            self.client_mapping[client_id] = sid
+
+            return conn
+
+    def _create_new_connection(
+        self, sid: str, client_id: str, now: float
+    ) -> ClientConnection:
+        """Create a new connection object."""
+        conn = ClientConnection(
+            sid=sid,
+            client_id=client_id,
+            state=ConnectionState.CONNECTED,
+            connected_at=now,
+        )
+        conn.metrics.connect_count = 1
+        self.logger.info(f"New client connected: {client_id} (sid: {sid})")
+        return conn
+
+    async def unregister_connection(self, sid: str, reason: str = "unknown") -> None:
+        """
+        Unregister a connection but keep state for reconnection.
+
+        Args:
+            sid: Socket ID
+            reason: Disconnection reason
+        """
+        async with self._lock:
+            if sid not in self.connections:
+                return
+
+            conn = self.connections[sid]
+            now = time.time()
+
+            # Update connection state
+            conn.state = ConnectionState.DISCONNECTED
+            conn.disconnected_at = now
+            conn.metrics.disconnect_count += 1
+
+            # Update uptime
+            if conn.connected_at:
+                conn.metrics.total_uptime += now - conn.connected_at
+
+            self.logger.info(
+                f"Client {conn.client_id} disconnected (sid: {sid}, reason: {reason}, "
+                f"buffered events: {len(conn.event_buffer)})"
+            )
+
+            # Keep connection for potential reconnection
+            # It will be cleaned up by health check if not reconnected
+
+    async def buffer_event(self, sid: str, event: Dict[str, Any]) -> bool:
+        """
+        Buffer an event for a client.
+
+        Args:
+            sid: Socket ID
+            event: Event to buffer
+
+        Returns:
+            True if buffered successfully
+        """
+        async with self._lock:
+            if sid not in self.connections:
+                return False
+
+            conn = self.connections[sid]
+
+            # Add sequence number
+            self.global_sequence += 1
+            event["sequence"] = self.global_sequence
+            event["timestamp"] = time.time()
+
+            # Buffer the event
+            conn.event_buffer.append(event)
+            conn.event_sequence = self.global_sequence
+            conn.metrics.events_buffered += 1
+
+            # Drop old events if buffer is full
+            if len(conn.event_buffer) >= self.max_buffer_size:
+                conn.metrics.events_dropped += 1
+
+            return True
+
+    async def get_replay_events(
+        self, sid: str, last_sequence: int = 0
+    ) -> List[Dict[str, Any]]:
+        """
+        Get events to replay for a client after reconnection.
+
+        Args:
+            sid: Socket ID
+            last_sequence: Last sequence number received by client
+
+        Returns:
+            List of events to replay
+        """
+        async with self._lock:
+            if sid not in self.connections:
+                return []
+
+            conn = self.connections[sid]
+            now = time.time()
+
+            # Filter events by sequence and TTL
+            replay_events = []
+            for event in conn.event_buffer:
+                if event.get("sequence", 0) > last_sequence:
+                    # Check TTL
+                    event_age = now - event.get("timestamp", 0)
+                    if event_age < self.event_ttl:
+                        replay_events.append(event)
+
+            self.logger.info(
+                f"Replaying {len(replay_events)} events for {conn.client_id} "
+                f"(from sequence {last_sequence})"
+            )
+
+            return replay_events
+
+    async def acknowledge_event(self, sid: str, sequence: int) -> None:
+        """
+        Acknowledge receipt of an event by a client.
+
+        Args:
+            sid: Socket ID
+            sequence: Sequence number of acknowledged event
+        """
+        async with self._lock:
+            if sid not in self.connections:
+                return
+
+            conn = self.connections[sid]
+            conn.last_acked_sequence = max(conn.last_acked_sequence, sequence)
+            conn.metrics.events_acked += 1
+
+            # Remove from pending acks
+            if sequence in conn.pending_acks:
+                del conn.pending_acks[sequence]
+
+    async def update_activity(self, sid: str, activity_type: str = "event") -> None:
+        """
+        Update last activity time for a connection.
+
+        Args:
+            sid: Socket ID
+            activity_type: Type of activity (event, ping, pong)
+        """
+        if sid not in self.connections:
+            return
+
+        conn = self.connections[sid]
+        now = time.time()
+
+        if activity_type == "ping":
+            conn.last_ping = now
+        elif activity_type == "pong":
+            conn.last_pong = now
+        else:
+            conn.last_event = now
+
+        conn.metrics.last_activity = now
+
+    async def start_health_monitoring(self) -> None:
+        """Start the health monitoring task."""
+        if self.health_task:
+            return
+
+        self.health_task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Started connection health monitoring")
+
+    async def stop_health_monitoring(self) -> None:
+        """Stop the health monitoring task."""
+        if self.health_task:
+            self.health_task.cancel()
+            try:
+                await self.health_task
+            except asyncio.CancelledError:
+                pass
+            self.health_task = None
+            self.logger.info("Stopped connection health monitoring")
+
+    async def _health_check_loop(self) -> None:
+        """Periodic health check for all connections."""
+        while True:
+            try:
+                await asyncio.sleep(self.health_check_interval)
+                await self.check_connection_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def check_connection_health(self) -> Dict[str, Any]:
+        """
+        Check health of all connections and clean up stale ones.
+
+        Returns:
+            Health status report
+        """
+        async with self._lock:
+            now = time.time()
+            report = {
+                "timestamp": datetime.now().isoformat(),
+                "total_connections": len(self.connections),
+                "healthy": 0,
+                "stale": 0,
+                "disconnected": 0,
+                "cleaned": 0,
+                "quality_scores": {},
+            }
+
+            to_clean = []
+
+            for sid, conn in self.connections.items():
+                # Calculate quality
+                quality = conn.calculate_quality()
+                report["quality_scores"][conn.client_id] = quality
+
+                if conn.state == ConnectionState.CONNECTED:
+                    if conn.is_healthy(self.stale_timeout):
+                        report["healthy"] += 1
+                    else:
+                        # Mark as stale
+                        conn.state = ConnectionState.STALE
+                        report["stale"] += 1
+                        self.logger.warning(
+                            f"Connection {conn.client_id} marked as stale "
+                            f"(last activity: {now - conn.metrics.last_activity:.1f}s ago)"
+                        )
+                elif conn.state == ConnectionState.DISCONNECTED:
+                    report["disconnected"] += 1
+
+                    # Clean up old disconnected connections
+                    if (
+                        conn.disconnected_at
+                        and (now - conn.disconnected_at) > self.event_ttl
+                    ):
+                        to_clean.append(sid)
+
+            # Clean up old connections
+            for sid in to_clean:
+                conn = self.connections[sid]
+                del self.connections[sid]
+                if conn.client_id in self.client_mapping:
+                    del self.client_mapping[conn.client_id]
+                report["cleaned"] += 1
+                self.logger.info(f"Cleaned up old connection: {conn.client_id}")
+
+            if report["stale"] > 0 or report["cleaned"] > 0:
+                self.logger.info(
+                    f"Health check: {report['healthy']} healthy, "
+                    f"{report['stale']} stale, {report['disconnected']} disconnected, "
+                    f"{report['cleaned']} cleaned"
+                )
+
+            return report
+
+    def get_connection(self, sid: str) -> Optional[ClientConnection]:
+        """Get connection by socket ID."""
+        return self.connections.get(sid)
+
+    def get_all_connections(self) -> Dict[str, ClientConnection]:
+        """Get all connections."""
+        return self.connections.copy()
+
+    def get_metrics(self) -> Dict[str, Any]:
+        """Get overall connection metrics."""
+        total_events_sent = sum(
+            c.metrics.events_sent for c in self.connections.values()
+        )
+        total_events_acked = sum(
+            c.metrics.events_acked for c in self.connections.values()
+        )
+        total_events_buffered = sum(
+            c.metrics.events_buffered for c in self.connections.values()
+        )
+        total_events_dropped = sum(
+            c.metrics.events_dropped for c in self.connections.values()
+        )
+        avg_quality = sum(
+            c.metrics.connection_quality for c in self.connections.values()
+        ) / max(len(self.connections), 1)
+
+        return {
+            "total_connections": len(self.connections),
+            "active_connections": sum(
+                1
+                for c in self.connections.values()
+                if c.state == ConnectionState.CONNECTED
+            ),
+            "total_events_sent": total_events_sent,
+            "total_events_acked": total_events_acked,
+            "total_events_buffered": total_events_buffered,
+            "total_events_dropped": total_events_dropped,
+            "average_quality": avg_quality,
+            "global_sequence": self.global_sequence,
+        }

claude_mpm/services/socketio/server/core.py

@@ -172,6 +172,9 @@ class SocketIOServerCore:
             self.app = web.Application()
             self.sio.attach(self.app)
 
+            # Setup HTTP API endpoints for receiving events from hook handlers
+            self._setup_http_api()
+
             # Find and serve static files
             self._setup_static_files()
 
@@ -229,6 +232,48 @@ class SocketIOServerCore:
         except Exception as e:
             self.logger.error(f"Error stopping Socket.IO server: {e}")
 
+    def _setup_http_api(self):
+        """Setup HTTP API endpoints for receiving events from hook handlers.
+
+        WHY: Hook handlers are ephemeral processes that spawn and die quickly.
+        Using HTTP POST allows them to send events without managing persistent
+        connections, eliminating disconnection issues.
+        """
+
+        async def api_events_handler(request):
+            """Handle POST /api/events from hook handlers."""
+            try:
+                # Parse JSON payload
+                event_data = await request.json()
+
+                # Log receipt if debugging
+                event_type = event_data.get("subtype", "unknown")
+                self.logger.debug(f"Received HTTP event: {event_type}")
+
+                # Broadcast to all connected dashboard clients via SocketIO
+                if self.sio:
+                    # The event is already in claude_event format from the hook handler
+                    await self.sio.emit("claude_event", event_data)
+
+                    # Update stats
+                    self.stats["events_sent"] = self.stats.get("events_sent", 0) + 1
+
+                    # Add to event buffer for late-joining clients
+                    with self.buffer_lock:
+                        self.event_buffer.append(event_data)
+                        self.stats["events_buffered"] = len(self.event_buffer)
+
+                # Return 204 No Content for success
+                return web.Response(status=204)
+
+            except Exception as e:
+                self.logger.error(f"Error handling HTTP event: {e}")
+                return web.Response(status=500, text=str(e))
+
+        # Register the HTTP POST endpoint
+        self.app.router.add_post("/api/events", api_events_handler)
+        self.logger.info("✅ HTTP API endpoint registered at /api/events")
+
     def _setup_static_files(self):
         """Setup static file serving for the dashboard."""
         try:
@@ -261,6 +306,24 @@ class SocketIOServerCore:
 
                 self.app.router.add_get("/", index_handler)
 
+                # Serve the actual dashboard template at /dashboard
+                async def dashboard_handler(request):
+                    dashboard_template = (
+                        self.dashboard_path.parent / "templates" / "index.html"
+                    )
+                    if dashboard_template.exists():
+                        self.logger.debug(
+                            f"Serving dashboard template from: {dashboard_template}"
+                        )
+                        return web.FileResponse(dashboard_template)
+                    # Fallback to the main index if template doesn't exist
+                    self.logger.warning(
+                        f"Dashboard template not found at: {dashboard_template}, falling back to index"
+                    )
+                    return await index_handler(request)
+
+                self.app.router.add_get("/dashboard", dashboard_handler)
+
                 # Serve version.json from dashboard directory
                 async def version_handler(request):
                     version_file = self.dashboard_path / "version.json"

claude_mpm/services/socketio/server/eventbus_integration.py

@@ -10,8 +10,9 @@ WHY this integration module:
 import logging
 from typing import Optional
 
-from claude_mpm.services.event_bus import EventBus, SocketIORelay
+from claude_mpm.services.event_bus import EventBus
 from claude_mpm.services.event_bus.config import get_config
+from claude_mpm.services.event_bus.direct_relay import DirectSocketIORelay
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +34,7 @@ class EventBusIntegration:
             server_instance: Optional Socket.IO server instance
         """
         self.server = server_instance
-        self.relay: Optional[SocketIORelay] = None
+        self.relay: Optional[DirectSocketIORelay] = None
         self.event_bus: Optional[EventBus] = None
         self.config = get_config()
         self.enabled = self.config.enabled and self.config.relay_enabled
@@ -76,23 +77,33 @@ class EventBusIntegration:
             # Apply configuration
             self.config.apply_to_eventbus(self.event_bus)
 
-            # Create and configure relay
-            relay_port = port or self.config.relay_port
+            # Create direct relay that uses server's broadcaster
             print(
-                f"[{datetime.now().isoformat()}] Creating SocketIORelay on port {relay_port}...",
+                f"[{datetime.now().isoformat()}] Creating DirectSocketIORelay...",
                 flush=True,
             )
-            self.relay = SocketIORelay(relay_port)
-            self.config.apply_to_relay(self.relay)
+            if self.server:
+                self.relay = DirectSocketIORelay(self.server)
+                print(
+                    f"[{datetime.now().isoformat()}] DirectSocketIORelay created with server instance",
+                    flush=True,
+                )
+            else:
+                logger.warning("No server instance provided, relay won't work")
+                print(
+                    f"[{datetime.now().isoformat()}] WARNING: No server instance for relay",
+                    flush=True,
+                )
+                return False
 
             # Start the relay
             print(f"[{datetime.now().isoformat()}] Starting relay...", flush=True)
             self.relay.start()
             print(f"[{datetime.now().isoformat()}] Relay started", flush=True)
 
-            logger.info(f"EventBus integration setup complete (port: {relay_port})")
+            logger.info("EventBus integration setup complete with DirectSocketIORelay")
             print(
-                f"[{datetime.now().isoformat()}] EventBus integration setup complete",
+                f"[{datetime.now().isoformat()}] EventBus integration setup complete with DirectSocketIORelay",
                 flush=True,
             )
             return True