claude-mpm 4.0.32__py3-none-any.whl → 4.1.0__py3-none-any.whl

claude_mpm/services/events/consumers/metrics.py

@@ -0,0 +1,242 @@
+"""
+Metrics Event Consumer
+=====================
+
+Collects metrics and statistics from events.
+"""
+
+import time
+from collections import defaultdict, deque
+from typing import Any, Deque, Dict, List
+
+from claude_mpm.core.logging_config import get_logger
+
+from ..core import Event
+from ..interfaces import ConsumerConfig, ConsumerPriority, IEventConsumer
+
+
+class MetricsConsumer(IEventConsumer):
+    """
+    Collects metrics and statistics from events.
+    
+    Features:
+    - Event counting by topic and type
+    - Rate calculation (events per second)
+    - Latency tracking
+    - Top event analysis
+    - Time-windowed statistics
+    """
+    
+    def __init__(
+        self,
+        window_size: int = 300,  # 5 minutes
+        top_n: int = 10,
+        report_interval: float = 60.0,  # Report every minute
+    ):
+        """
+        Initialize metrics consumer.
+        
+        Args:
+            window_size: Time window for statistics (seconds)
+            top_n: Number of top events to track
+            report_interval: How often to report metrics (seconds)
+        """
+        self.logger = get_logger("MetricsConsumer")
+        
+        # Configuration
+        self.window_size = window_size
+        self.top_n = top_n
+        self.report_interval = report_interval
+        
+        # State
+        self._initialized = False
+        self._last_report_time = time.time()
+        
+        # Metrics storage
+        self._event_counts: Dict[str, int] = defaultdict(int)
+        self._topic_counts: Dict[str, int] = defaultdict(int)
+        self._type_counts: Dict[str, int] = defaultdict(int)
+        self._source_counts: Dict[str, int] = defaultdict(int)
+        
+        # Time-windowed metrics
+        self._recent_events: Deque[tuple] = deque()  # (timestamp, topic, type)
+        self._latencies: Deque[float] = deque(maxlen=1000)
+        
+        # Error tracking
+        self._error_counts: Dict[str, int] = defaultdict(int)
+        
+        # Performance metrics
+        self._metrics = {
+            "total_events": 0,
+            "events_per_second": 0.0,
+            "average_latency_ms": 0.0,
+            "peak_rate": 0.0,
+            "unique_topics": 0,
+            "unique_types": 0,
+        }
+        
+        # Consumer configuration
+        self._config = ConsumerConfig(
+            name="MetricsConsumer",
+            topics=["**"],  # Monitor all events
+            priority=ConsumerPriority.DEFERRED,  # Process after other consumers
+        )
+    
+    async def initialize(self) -> bool:
+        """Initialize the metrics consumer."""
+        self._initialized = True
+        self.logger.info("Metrics consumer initialized")
+        return True
+    
+    async def consume(self, event: Event) -> bool:
+        """Process a single event for metrics."""
+        if not self._initialized:
+            return False
+        
+        try:
+            current_time = time.time()
+            
+            # Update counts
+            self._event_counts[f"{event.topic}:{event.type}"] += 1
+            self._topic_counts[event.topic] += 1
+            self._type_counts[event.type] += 1
+            self._source_counts[event.source] += 1
+            self._metrics["total_events"] += 1
+            
+            # Track errors
+            if event.metadata and event.metadata.consumers_failed:
+                for consumer in event.metadata.consumers_failed:
+                    self._error_counts[consumer] += 1
+            
+            # Add to recent events
+            self._recent_events.append((current_time, event.topic, event.type))
+            
+            # Calculate latency if timestamp available
+            if event.timestamp:
+                latency = (current_time - event.timestamp.timestamp()) * 1000
+                self._latencies.append(latency)
+            
+            # Clean old events from window
+            cutoff_time = current_time - self.window_size
+            while self._recent_events and self._recent_events[0][0] < cutoff_time:
+                self._recent_events.popleft()
+            
+            # Report metrics periodically
+            if current_time - self._last_report_time >= self.report_interval:
+                await self._report_metrics()
+                self._last_report_time = current_time
+            
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"Error processing event for metrics: {e}")
+            return False
+    
+    async def consume_batch(self, events: List[Event]) -> int:
+        """Process multiple events."""
+        successful = 0
+        for event in events:
+            if await self.consume(event):
+                successful += 1
+        return successful
+    
+    async def shutdown(self) -> None:
+        """Shutdown the consumer."""
+        # Report final metrics
+        await self._report_metrics()
+        
+        self.logger.info(
+            f"Metrics consumer shutdown - processed {self._metrics['total_events']} events"
+        )
+        self._initialized = False
+    
+    @property
+    def config(self) -> ConsumerConfig:
+        """Get consumer configuration."""
+        return self._config
+    
+    @property
+    def is_healthy(self) -> bool:
+        """Check if consumer is healthy."""
+        return self._initialized
+    
+    def get_metrics(self) -> Dict[str, Any]:
+        """Get consumer metrics."""
+        # Calculate current metrics
+        self._calculate_metrics()
+        
+        # Get top events
+        top_events = sorted(
+            self._event_counts.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )[:self.top_n]
+        
+        top_topics = sorted(
+            self._topic_counts.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )[:self.top_n]
+        
+        return {
+            **self._metrics,
+            "top_events": dict(top_events),
+            "top_topics": dict(top_topics),
+            "error_counts": dict(self._error_counts),
+            "window_size_seconds": self.window_size,
+        }
+    
+    def _calculate_metrics(self) -> None:
+        """Calculate current metrics."""
+        # Events per second
+        if self._recent_events:
+            time_span = time.time() - self._recent_events[0][0]
+            if time_span > 0:
+                self._metrics["events_per_second"] = len(self._recent_events) / time_span
+        
+        # Average latency
+        if self._latencies:
+            self._metrics["average_latency_ms"] = sum(self._latencies) / len(self._latencies)
+        
+        # Unique counts
+        self._metrics["unique_topics"] = len(self._topic_counts)
+        self._metrics["unique_types"] = len(self._type_counts)
+        
+        # Peak rate
+        if self._metrics["events_per_second"] > self._metrics["peak_rate"]:
+            self._metrics["peak_rate"] = self._metrics["events_per_second"]
+    
+    async def _report_metrics(self) -> None:
+        """Report current metrics to log."""
+        self._calculate_metrics()
+        
+        # Build report
+        report = [
+            f"=== Event Metrics Report ===",
+            f"Total Events: {self._metrics['total_events']}",
+            f"Rate: {self._metrics['events_per_second']:.2f} events/sec",
+            f"Avg Latency: {self._metrics['average_latency_ms']:.1f}ms",
+            f"Unique Topics: {self._metrics['unique_topics']}",
+            f"Unique Types: {self._metrics['unique_types']}",
+        ]
+        
+        # Add top events
+        top_events = sorted(
+            self._event_counts.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )[:5]
+        
+        if top_events:
+            report.append("\nTop Events:")
+            for event_key, count in top_events:
+                report.append(f"  {event_key}: {count}")
+        
+        # Add error summary
+        if self._error_counts:
+            report.append("\nErrors by Consumer:")
+            for consumer, count in self._error_counts.items():
+                report.append(f"  {consumer}: {count}")
+        
+        # Log report
+        self.logger.info("\n".join(report))

claude_mpm/services/events/consumers/socketio.py

@@ -0,0 +1,376 @@
+"""
+Socket.IO Event Consumer
+=======================
+
+Consumes events from the event bus and emits them via Socket.IO.
+This is the only component that knows about Socket.IO, decoupling
+the rest of the system from the transport layer.
+"""
+
+import asyncio
+import time
+from typing import Any, Dict, List, Optional
+
+from claude_mpm.core.logging_config import get_logger
+
+from ..core import Event
+from ..interfaces import ConsumerConfig, ConsumerPriority, IEventConsumer
+
+
+class SocketIOConsumer(IEventConsumer):
+    """
+    Consumes events and emits them via Socket.IO.
+    
+    Features:
+    - Single Socket.IO connection management
+    - Automatic reconnection with backoff
+    - Event batching for efficiency
+    - Connection health monitoring
+    - Graceful degradation when Socket.IO unavailable
+    """
+    
+    def __init__(
+        self,
+        socketio_server=None,
+        port_range: tuple = (8765, 8785),
+        reconnect_delay: float = 1.0,
+        max_reconnect_delay: float = 30.0,
+        batch_size: int = 10,
+        batch_timeout: float = 0.1,
+    ):
+        """
+        Initialize Socket.IO consumer.
+        
+        Args:
+            socketio_server: Socket.IO server instance (optional)
+            port_range: Port range to try for connection
+            reconnect_delay: Initial reconnection delay
+            max_reconnect_delay: Maximum reconnection delay
+            batch_size: Number of events to batch
+            batch_timeout: Max time to wait for batch
+        """
+        self.logger = get_logger("SocketIOConsumer")
+        
+        # Socket.IO configuration
+        self.socketio_server = socketio_server
+        self.port_range = port_range
+        self.reconnect_delay = reconnect_delay
+        self.max_reconnect_delay = max_reconnect_delay
+        self.current_reconnect_delay = reconnect_delay
+        
+        # Batching configuration
+        self.batch_size = batch_size
+        self.batch_timeout = batch_timeout
+        
+        # State
+        self._initialized = False
+        self._connected = False
+        self._reconnect_task: Optional[asyncio.Task] = None
+        self._event_batch: List[Event] = []
+        self._batch_timer: Optional[asyncio.Task] = None
+        
+        # Metrics
+        self._metrics = {
+            "events_received": 0,
+            "events_emitted": 0,
+            "events_failed": 0,
+            "connection_failures": 0,
+            "reconnections": 0,
+            "last_emit_time": None,
+            "average_emit_time_ms": 0,
+        }
+        
+        # Consumer configuration
+        self._config = ConsumerConfig(
+            name="SocketIOConsumer",
+            topics=["**"],  # Subscribe to all events
+            priority=ConsumerPriority.HIGH,
+            batch_size=batch_size,
+            batch_timeout=batch_timeout,
+            max_retries=3,
+            retry_backoff=2.0,
+        )
+    
+    async def initialize(self) -> bool:
+        """Initialize the Socket.IO consumer."""
+        if self._initialized:
+            return True
+        
+        self.logger.info("Initializing Socket.IO consumer")
+        
+        # Try to import socketio if not provided
+        if self.socketio_server is None:
+            try:
+                # Try to get existing server instance
+                from claude_mpm.services.socketio.server import get_socketio_server
+                self.socketio_server = get_socketio_server()
+                
+                if self.socketio_server:
+                    self._connected = True
+                    self.logger.info("Connected to existing Socket.IO server")
+                
+            except ImportError:
+                self.logger.warning("Socket.IO server not available")
+                # Continue without Socket.IO - events will be dropped
+        
+        self._initialized = True
+        
+        # Start reconnection task if not connected
+        if not self._connected and self.socketio_server:
+            self._reconnect_task = asyncio.create_task(self._reconnect_loop())
+        
+        return True
+    
+    async def consume(self, event: Event) -> bool:
+        """
+        Process a single event.
+        
+        Events are batched for efficiency and emitted via Socket.IO.
+        """
+        if not self._initialized:
+            self.logger.warning("Consumer not initialized")
+            return False
+        
+        self._metrics["events_received"] += 1
+        
+        # Add to batch
+        self._event_batch.append(event)
+        
+        # Process batch if full
+        if len(self._event_batch) >= self.batch_size:
+            return await self._flush_batch()
+        
+        # Start batch timer if not running
+        if self._batch_timer is None or self._batch_timer.done():
+            self._batch_timer = asyncio.create_task(self._batch_timeout_handler())
+        
+        return True
+    
+    async def consume_batch(self, events: List[Event]) -> int:
+        """Process multiple events in a batch."""
+        if not self._initialized:
+            return 0
+        
+        successful = 0
+        for event in events:
+            if await self.consume(event):
+                successful += 1
+        
+        return successful
+    
+    async def shutdown(self) -> None:
+        """Shutdown the consumer gracefully."""
+        self.logger.info("Shutting down Socket.IO consumer")
+        
+        # Cancel reconnection task
+        if self._reconnect_task:
+            self._reconnect_task.cancel()
+            try:
+                await self._reconnect_task
+            except asyncio.CancelledError:
+                pass
+        
+        # Cancel batch timer
+        if self._batch_timer:
+            self._batch_timer.cancel()
+            try:
+                await self._batch_timer
+            except asyncio.CancelledError:
+                pass
+        
+        # Flush remaining events
+        if self._event_batch:
+            await self._flush_batch()
+        
+        self._initialized = False
+        self._connected = False
+        
+        self.logger.info("Socket.IO consumer shutdown complete")
+    
+    @property
+    def config(self) -> ConsumerConfig:
+        """Get consumer configuration."""
+        return self._config
+    
+    @property
+    def is_healthy(self) -> bool:
+        """Check if consumer is healthy."""
+        # Healthy if initialized and either connected or attempting to reconnect
+        return self._initialized and (self._connected or self._reconnect_task is not None)
+    
+    def get_metrics(self) -> Dict[str, Any]:
+        """Get consumer metrics."""
+        return {
+            **self._metrics,
+            "connected": self._connected,
+            "batch_size": len(self._event_batch),
+        }
+    
+    async def _flush_batch(self) -> bool:
+        """
+        Flush the current batch of events to Socket.IO.
+        
+        Returns:
+            True if all events emitted successfully
+        """
+        if not self._event_batch:
+            return True
+        
+        batch = self._event_batch
+        self._event_batch = []
+        
+        # Cancel batch timer
+        if self._batch_timer:
+            self._batch_timer.cancel()
+            self._batch_timer = None
+        
+        # Emit events
+        success = await self._emit_events(batch)
+        
+        if not success:
+            # Re-queue failed events
+            self._event_batch = batch + self._event_batch
+            return False
+        
+        return True
+    
+    async def _emit_events(self, events: List[Event]) -> bool:
+        """
+        Emit events via Socket.IO.
+        
+        Args:
+            events: Events to emit
+            
+        Returns:
+            True if all events emitted successfully
+        """
+        if not self._connected or not self.socketio_server:
+            self.logger.debug(f"Cannot emit {len(events)} events - not connected")
+            self._metrics["events_failed"] += len(events)
+            return False
+        
+        try:
+            start_time = time.time()
+            
+            for event in events:
+                # Convert event to Socket.IO format
+                socketio_event = self._convert_to_socketio(event)
+                
+                # Emit event
+                await self.socketio_server.emit(
+                    socketio_event["event"],
+                    socketio_event["data"],
+                    namespace=socketio_event.get("namespace", "/")
+                )
+                
+                self._metrics["events_emitted"] += 1
+            
+            # Update metrics
+            elapsed_ms = (time.time() - start_time) * 1000
+            self._metrics["last_emit_time"] = time.time()
+            
+            # Update rolling average
+            avg = self._metrics["average_emit_time_ms"]
+            self._metrics["average_emit_time_ms"] = (avg * 0.9) + (elapsed_ms * 0.1)
+            
+            self.logger.debug(
+                f"Emitted {len(events)} events in {elapsed_ms:.1f}ms "
+                f"(avg: {self._metrics['average_emit_time_ms']:.1f}ms)"
+            )
+            
+            # Reset reconnect delay on success
+            self.current_reconnect_delay = self.reconnect_delay
+            
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"Error emitting events: {e}")
+            self._metrics["events_failed"] += len(events)
+            self._metrics["connection_failures"] += 1
+            self._connected = False
+            
+            # Start reconnection
+            if self._reconnect_task is None or self._reconnect_task.done():
+                self._reconnect_task = asyncio.create_task(self._reconnect_loop())
+            
+            return False
+    
+    def _convert_to_socketio(self, event: Event) -> Dict[str, Any]:
+        """
+        Convert an Event to Socket.IO format.
+        
+        Maps our standard event format to Socket.IO's expected format.
+        """
+        # Determine Socket.IO event name based on topic
+        if event.topic.startswith("hook."):
+            socketio_event = "hook_event"
+        elif event.topic.startswith("cli."):
+            socketio_event = "cli_event"
+        elif event.topic.startswith("system."):
+            socketio_event = "system_event"
+        elif event.topic.startswith("agent."):
+            socketio_event = "agent_event"
+        elif event.topic.startswith("build."):
+            socketio_event = "build_event"
+        else:
+            socketio_event = "claude_event"
+        
+        # Build Socket.IO data
+        return {
+            "event": socketio_event,
+            "data": {
+                "id": event.id,
+                "type": event.type,
+                "topic": event.topic,
+                "timestamp": event.timestamp.isoformat(),
+                "source": event.source,
+                "data": event.data,
+                "correlation_id": event.correlation_id,
+            },
+            "namespace": "/",
+        }
+    
+    async def _batch_timeout_handler(self) -> None:
+        """Handle batch timeout - flush partial batch."""
+        await asyncio.sleep(self.batch_timeout)
+        
+        if self._event_batch:
+            self.logger.debug(f"Batch timeout - flushing {len(self._event_batch)} events")
+            await self._flush_batch()
+    
+    async def _reconnect_loop(self) -> None:
+        """
+        Reconnection loop with exponential backoff.
+        
+        Attempts to reconnect to Socket.IO server when connection is lost.
+        """
+        while not self._connected and self._initialized:
+            try:
+                self.logger.info(
+                    f"Attempting to reconnect to Socket.IO "
+                    f"(delay: {self.current_reconnect_delay}s)"
+                )
+                
+                # Try to reconnect
+                if self.socketio_server:
+                    # Test connection
+                    # In real implementation, would check server status
+                    self._connected = True
+                    self._metrics["reconnections"] += 1
+                    self.logger.info("Reconnected to Socket.IO server")
+                    
+                    # Reset delay
+                    self.current_reconnect_delay = self.reconnect_delay
+                    break
+                
+            except Exception as e:
+                self.logger.error(f"Reconnection failed: {e}")
+            
+            # Wait before next attempt
+            await asyncio.sleep(self.current_reconnect_delay)
+            
+            # Exponential backoff
+            self.current_reconnect_delay = min(
+                self.current_reconnect_delay * 2,
+                self.max_reconnect_delay
+            )