claude-mpm 4.13.2__py3-none-any.whl → 4.18.2__py3-none-any.whl

claude_mpm/services/local_ops/log_monitor.py

@@ -0,0 +1,396 @@
+"""
+Log Monitor for Claude MPM Framework
+======================================
+
+WHY: Provides real-time log file monitoring to detect error patterns that
+indicate imminent crashes (OOM, exceptions, segfaults) BEFORE they occur.
+
+DESIGN DECISION: Uses watchdog library for efficient OS-level file system
+monitoring. Avoids polling by receiving file modification events from the OS.
+
+ARCHITECTURE:
+- Watchdog-based file system monitoring (OS-level events)
+- Regex-based pattern matching for error detection
+- Configurable error patterns with severity levels
+- Rolling window of recent matches per deployment
+- Callback system for pattern match alerts
+- Thread-safe with proper locking
+
+USAGE:
+    monitor = LogMonitor()
+    monitor.initialize()
+
+    # Add error patterns
+    monitor.add_pattern(r"OutOfMemoryError", severity="CRITICAL")
+    monitor.add_pattern(r"Exception:", severity="ERROR")
+
+    # Start monitoring a log file
+    monitor.start_monitoring(
+        log_file="/var/log/app.log",
+        deployment_id="my-app"
+    )
+
+    # Get recent matches
+    matches = monitor.get_recent_matches(deployment_id, limit=10)
+"""
+
+import re
+import threading
+from collections import defaultdict
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Tuple
+
+from watchdog.events import FileSystemEvent, FileSystemEventHandler
+from watchdog.observers import Observer
+
+from claude_mpm.core.logger import get_logger
+from claude_mpm.services.core.base import SyncBaseService
+from claude_mpm.services.core.interfaces.stability import ILogMonitor
+from claude_mpm.services.core.models.stability import LogPatternMatch
+
+
+class LogFileHandler(FileSystemEventHandler):
+    """
+    File system event handler for log file monitoring.
+
+    WHY: Receives OS-level file modification events and triggers
+    pattern matching on new log lines.
+    """
+
+    def __init__(
+        self,
+        log_file: str,
+        deployment_id: str,
+        on_new_lines: Callable[[str, List[str]], None],
+    ):
+        """
+        Initialize log file handler.
+
+        Args:
+            log_file: Path to log file being monitored
+            deployment_id: Deployment identifier
+            on_new_lines: Callback for new lines: (deployment_id, lines)
+        """
+        super().__init__()
+        self.log_file = Path(log_file).resolve()
+        self.deployment_id = deployment_id
+        self.on_new_lines = on_new_lines
+        self.last_position = 0
+
+        # Initialize to end of file
+        if self.log_file.exists():
+            self.last_position = self.log_file.stat().st_size
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        """
+        Handle file modification events.
+
+        Args:
+            event: File system event
+        """
+        if event.is_directory:
+            return
+
+        # Check if this is our log file
+        event_path = Path(event.src_path).resolve()
+        if event_path != self.log_file:
+            return
+
+        # Read new lines
+        try:
+            new_lines = self._read_new_lines()
+            if new_lines:
+                self.on_new_lines(self.deployment_id, new_lines)
+        except Exception as e:
+            # Log error but don't crash the monitoring thread
+            get_logger().error(f"Error reading new log lines: {e}")
+
+    def _read_new_lines(self) -> List[str]:
+        """
+        Read new lines from log file since last read.
+
+        Returns:
+            List of new lines
+        """
+        if not self.log_file.exists():
+            return []
+
+        new_lines = []
+
+        with Path(self.log_file).open(encoding="utf-8", errors="ignore") as f:
+            # Seek to last position
+            f.seek(self.last_position)
+
+            # Read new lines
+            for line in f:
+                new_lines.append(line.rstrip("\n\r"))
+
+            # Update position
+            self.last_position = f.tell()
+
+        return new_lines
+
+
+class LogMonitor(SyncBaseService, ILogMonitor):
+    """
+    Real-time log file monitoring service.
+
+    WHY: Provides early warning of critical errors by monitoring log files
+    in real-time and detecting patterns that indicate imminent failures.
+
+    Thread Safety: All public methods are thread-safe with proper locking.
+    """
+
+    # Default error patterns
+    DEFAULT_PATTERNS = [
+        (r"OutOfMemoryError", "CRITICAL"),
+        (r"Segmentation fault", "CRITICAL"),
+        (r"Exception:", "ERROR"),
+        (r"Traceback", "ERROR"),
+        (r"Error:", "ERROR"),
+        (r"FATAL", "CRITICAL"),
+        (r"Database connection failed", "ERROR"),
+        (r"Connection refused", "WARNING"),
+        (r"Connection timeout", "WARNING"),
+    ]
+
+    def __init__(self, match_history_limit: int = 100):
+        """
+        Initialize log monitor.
+
+        Args:
+            match_history_limit: Number of matches to keep per deployment (default: 100)
+        """
+        super().__init__("LogMonitor")
+        self.match_history_limit = match_history_limit
+
+        # Error patterns: List[(pattern, severity)]
+        self._patterns: List[Tuple[re.Pattern, str]] = []
+
+        # Add default patterns
+        for pattern, severity in self.DEFAULT_PATTERNS:
+            self._patterns.append((re.compile(pattern), severity))
+
+        # Watchdog observer and handlers
+        self._observer: Optional[Observer] = None
+        self._handlers: Dict[str, LogFileHandler] = {}  # deployment_id -> handler
+
+        # Match history: deployment_id -> List[LogPatternMatch]
+        self._match_history: Dict[str, List[LogPatternMatch]] = defaultdict(list)
+
+        # Match callbacks
+        self._match_callbacks: List[Callable[[str, LogPatternMatch], None]] = []
+
+        # Thread safety
+        self._lock = threading.Lock()
+
+    def initialize(self) -> bool:
+        """
+        Initialize the log monitor.
+
+        Returns:
+            True if initialization successful
+        """
+        try:
+            # Create watchdog observer
+            self._observer = Observer()
+            self._observer.start()
+
+            self._initialized = True
+            self.log_info(
+                f"Log monitor initialized with {len(self._patterns)} patterns"
+            )
+            return True
+
+        except Exception as e:
+            self.log_error(f"Failed to initialize: {e}")
+            return False
+
+    def shutdown(self) -> None:
+        """Shutdown log monitor and stop all monitoring."""
+        # Stop all monitoring
+        with self._lock:
+            deployment_ids = list(self._handlers.keys())
+
+        for deployment_id in deployment_ids:
+            self.stop_monitoring(deployment_id)
+
+        # Stop observer
+        if self._observer:
+            self._observer.stop()
+            self._observer.join(timeout=5.0)
+
+        self._shutdown = True
+        self.log_info("Log monitor shutdown complete")
+
+    def start_monitoring(self, log_file: str, deployment_id: str) -> None:
+        """
+        Start monitoring a log file for error patterns.
+
+        WHY: Begins watching the log file for new entries. Uses OS-level
+        file system events for efficiency.
+
+        Args:
+            log_file: Path to log file to monitor
+            deployment_id: Deployment identifier for callbacks
+        """
+        log_path = Path(log_file).resolve()
+
+        if not log_path.exists():
+            self.log_warning(f"Log file does not exist: {log_file}")
+            # Still create handler - it will start monitoring when file is created
+            # return
+
+        with self._lock:
+            # Check if already monitoring
+            if deployment_id in self._handlers:
+                self.log_warning(f"Already monitoring logs for {deployment_id}")
+                return
+
+            # Create handler
+            handler = LogFileHandler(
+                log_file=str(log_path),
+                deployment_id=deployment_id,
+                on_new_lines=self._process_new_lines,
+            )
+
+            # Schedule handler with observer
+            if self._observer:
+                # Watch the directory containing the log file
+                watch_dir = log_path.parent
+                self._observer.schedule(handler, str(watch_dir), recursive=False)
+
+            self._handlers[deployment_id] = handler
+
+        self.log_info(f"Started monitoring log file for {deployment_id}: {log_file}")
+
+    def stop_monitoring(self, deployment_id: str) -> None:
+        """
+        Stop monitoring a deployment's log file.
+
+        Args:
+            deployment_id: Deployment identifier
+        """
+        with self._lock:
+            handler = self._handlers.pop(deployment_id, None)
+            if handler and self._observer:
+                # Unschedule handler
+                self._observer.unschedule_all()
+
+                # Reschedule remaining handlers
+                for remaining_handler in self._handlers.values():
+                    watch_dir = remaining_handler.log_file.parent
+                    self._observer.schedule(
+                        remaining_handler, str(watch_dir), recursive=False
+                    )
+
+        if handler:
+            self.log_info(f"Stopped monitoring logs for {deployment_id}")
+
+    def add_pattern(self, pattern: str, severity: str = "ERROR") -> None:
+        """
+        Add an error pattern to monitor.
+
+        Args:
+            pattern: Regex pattern to match
+            severity: Error severity (ERROR, CRITICAL, WARNING)
+        """
+        with self._lock:
+            compiled_pattern = re.compile(pattern)
+            self._patterns.append((compiled_pattern, severity))
+
+        self.log_debug(f"Added pattern: {pattern} (severity: {severity})")
+
+    def get_recent_matches(
+        self, deployment_id: str, limit: int = 10
+    ) -> List[LogPatternMatch]:
+        """
+        Get recent pattern matches for a deployment.
+
+        Args:
+            deployment_id: Deployment identifier
+            limit: Maximum number of matches to return
+
+        Returns:
+            List of LogPatternMatch objects, newest first
+        """
+        with self._lock:
+            matches = self._match_history.get(deployment_id, [])
+            return list(reversed(matches[-limit:]))
+
+    def register_match_callback(
+        self, callback: Callable[[str, LogPatternMatch], None]
+    ) -> None:
+        """
+        Register callback for pattern matches.
+
+        Args:
+            callback: Function called with (deployment_id, match) when pattern detected
+        """
+        with self._lock:
+            self._match_callbacks.append(callback)
+            self.log_debug(f"Registered match callback: {callback.__name__}")
+
+    def _process_new_lines(self, deployment_id: str, lines: List[str]) -> None:
+        """
+        Process new log lines for pattern matching.
+
+        Args:
+            deployment_id: Deployment identifier
+            lines: New log lines to process
+        """
+        for line in lines:
+            # Check against all patterns
+            for pattern, severity in self._patterns:
+                if pattern.search(line):
+                    # Create match
+                    match = LogPatternMatch(
+                        deployment_id=deployment_id,
+                        pattern=pattern.pattern,
+                        line=line,
+                        severity=severity,
+                    )
+
+                    # Add to history
+                    with self._lock:
+                        self._match_history[deployment_id].append(match)
+
+                        # Trim history
+                        if (
+                            len(self._match_history[deployment_id])
+                            > self.match_history_limit
+                        ):
+                            self._match_history[deployment_id] = self._match_history[
+                                deployment_id
+                            ][-self.match_history_limit :]
+
+                    # Log match
+                    self.log_warning(
+                        f"Pattern matched in {deployment_id}: "
+                        f"[{severity}] {pattern.pattern[:50]}"
+                    )
+
+                    # Trigger callbacks
+                    self._trigger_match_callbacks(deployment_id, match)
+
+                    # Only match first pattern per line
+                    break
+
+    def _trigger_match_callbacks(
+        self, deployment_id: str, match: LogPatternMatch
+    ) -> None:
+        """
+        Trigger registered callbacks for pattern matches.
+
+        Args:
+            deployment_id: Deployment that has a match
+            match: LogPatternMatch with pattern details
+        """
+        for callback in self._match_callbacks:
+            try:
+                callback(deployment_id, match)
+            except Exception as e:
+                self.log_error(f"Error in match callback {callback.__name__}: {e}")
+
+
+__all__ = ["LogMonitor"]

claude_mpm/services/local_ops/memory_leak_detector.py

@@ -0,0 +1,294 @@
+"""
+Memory Leak Detector for Claude MPM Framework
+==============================================
+
+WHY: Detects memory leaks BEFORE they cause OOM crashes by analyzing memory
+usage trends over time using linear regression slope analysis.
+
+DESIGN DECISION: Uses rolling window of memory measurements with configurable
+size and threshold. Calculates slope to detect sustained memory growth patterns.
+
+ARCHITECTURE:
+- Rolling window of (timestamp, memory_mb) measurements per deployment
+- Slope-based leak detection: MB/minute growth rate
+- Configurable thresholds and window sizes
+- Callback system for leak detection alerts
+- Thread-safe with proper locking
+
+USAGE:
+    detector = MemoryLeakDetector(
+        leak_threshold_mb_per_minute=10.0,
+        window_size=100,
+    )
+    detector.initialize()
+
+    # Record memory usage periodically
+    detector.record_memory_usage(deployment_id, memory_mb)
+
+    # Check for leaks
+    trend = detector.analyze_trend(deployment_id)
+    if trend.is_leaking:
+        print(f"Leak detected! Slope: {trend.slope_mb_per_minute} MB/min")
+"""
+
+import threading
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Callable, Dict, List, Tuple
+
+from claude_mpm.services.core.base import SyncBaseService
+from claude_mpm.services.core.interfaces.stability import IMemoryLeakDetector
+from claude_mpm.services.core.models.stability import MemoryTrend
+
+
+class MemoryLeakDetector(SyncBaseService, IMemoryLeakDetector):
+    """
+    Memory leak detection service using trend analysis.
+
+    WHY: Provides early warning of memory leaks by analyzing memory growth
+    patterns over time, enabling preemptive restarts before OOM crashes.
+
+    Algorithm:
+    1. Maintain rolling window of memory measurements
+    2. Calculate linear slope (MB per minute)
+    3. Detect leak if slope exceeds threshold (default: 10 MB/min)
+
+    Thread Safety: All public methods are thread-safe with proper locking.
+    """
+
+    def __init__(
+        self,
+        leak_threshold_mb_per_minute: float = 10.0,
+        window_size: int = 100,
+    ):
+        """
+        Initialize memory leak detector.
+
+        Args:
+            leak_threshold_mb_per_minute: Threshold for leak detection (default: 10.0)
+            window_size: Number of measurements to keep in rolling window (default: 100)
+        """
+        super().__init__("MemoryLeakDetector")
+        self.leak_threshold = leak_threshold_mb_per_minute
+        self.window_size = window_size
+
+        # Memory measurements: deployment_id -> List[(timestamp, memory_mb)]
+        self._measurements: Dict[str, List[Tuple[datetime, float]]] = defaultdict(list)
+
+        # Thread safety
+        self._lock = threading.Lock()
+
+        # Leak detection callbacks
+        self._leak_callbacks: List[Callable[[str, MemoryTrend], None]] = []
+
+    def initialize(self) -> bool:
+        """
+        Initialize the memory leak detector.
+
+        Returns:
+            True if initialization successful
+        """
+        self._initialized = True
+        self.log_info(
+            f"Memory leak detector initialized "
+            f"(threshold={self.leak_threshold} MB/min, window={self.window_size})"
+        )
+        return True
+
+    def shutdown(self) -> None:
+        """Shutdown memory leak detector and clear data."""
+        with self._lock:
+            self._measurements.clear()
+            self._leak_callbacks.clear()
+
+        self._shutdown = True
+        self.log_info("Memory leak detector shutdown complete")
+
+    def record_memory_usage(self, deployment_id: str, memory_mb: float) -> None:
+        """
+        Record a memory usage measurement.
+
+        WHY: Builds historical data for trend analysis. Should be called
+        periodically (e.g., every 30s) to collect sufficient data points.
+
+        Args:
+            deployment_id: Deployment identifier
+            memory_mb: Current memory usage in megabytes
+        """
+        with self._lock:
+            # Add new measurement
+            timestamp = datetime.now(tz=timezone.utc)
+            self._measurements[deployment_id].append((timestamp, memory_mb))
+
+            # Trim to window size
+            if len(self._measurements[deployment_id]) > self.window_size:
+                self._measurements[deployment_id] = self._measurements[deployment_id][
+                    -self.window_size :
+                ]
+
+            self.log_debug(
+                f"Recorded memory usage for {deployment_id}: {memory_mb:.2f}MB "
+                f"({len(self._measurements[deployment_id])} measurements)"
+            )
+
+    def analyze_trend(self, deployment_id: str) -> MemoryTrend:
+        """
+        Analyze memory usage trend for leak detection.
+
+        WHY: Computes slope of memory usage over time to detect sustained
+        growth patterns characteristic of memory leaks.
+
+        Args:
+            deployment_id: Deployment identifier
+
+        Returns:
+            MemoryTrend with slope analysis and leak detection result
+
+        Algorithm:
+            slope_mb_per_minute = (recent_memory - old_memory) / time_delta_minutes
+            is_leaking = slope_mb_per_minute > threshold
+        """
+        with self._lock:
+            measurements = self._measurements.get(deployment_id, [])
+
+            # Need at least 2 measurements for trend analysis
+            if len(measurements) < 2:
+                return MemoryTrend(
+                    deployment_id=deployment_id,
+                    timestamps=[],
+                    memory_mb=[],
+                    slope_mb_per_minute=0.0,
+                    is_leaking=False,
+                    window_size=0,
+                    threshold_mb_per_minute=self.leak_threshold,
+                )
+
+            # Extract timestamps and memory values
+            timestamps = [ts for ts, _ in measurements]
+            memory_mb = [mem for _, mem in measurements]
+
+            # Calculate slope using simple linear trend
+            slope = self._calculate_slope(measurements)
+
+            # Detect leak if slope exceeds threshold
+            is_leaking = slope > self.leak_threshold
+
+            trend = MemoryTrend(
+                deployment_id=deployment_id,
+                timestamps=timestamps,
+                memory_mb=memory_mb,
+                slope_mb_per_minute=slope,
+                is_leaking=is_leaking,
+                window_size=len(measurements),
+                threshold_mb_per_minute=self.leak_threshold,
+            )
+
+            # Trigger callbacks if leak detected
+            if is_leaking:
+                self.log_warning(
+                    f"Memory leak detected for {deployment_id}: "
+                    f"{slope:.2f} MB/min (threshold: {self.leak_threshold} MB/min)"
+                )
+                self._trigger_leak_callbacks(deployment_id, trend)
+
+            return trend
+
+    def is_leaking(self, deployment_id: str) -> bool:
+        """
+        Check if deployment has a detected memory leak.
+
+        Returns:
+            True if leak detected (sustained memory growth)
+        """
+        trend = self.analyze_trend(deployment_id)
+        return trend.is_leaking
+
+    def register_leak_callback(
+        self, callback: Callable[[str, MemoryTrend], None]
+    ) -> None:
+        """
+        Register callback for leak detection events.
+
+        Args:
+            callback: Function called with (deployment_id, trend) when leak detected
+        """
+        with self._lock:
+            self._leak_callbacks.append(callback)
+            self.log_debug(f"Registered leak callback: {callback.__name__}")
+
+    def _calculate_slope(self, measurements: List[Tuple[datetime, float]]) -> float:
+        """
+        Calculate memory growth slope using simple linear regression.
+
+        WHY: Linear slope provides a robust measure of sustained memory growth,
+        filtering out normal variations and temporary spikes.
+
+        Args:
+            measurements: List of (timestamp, memory_mb) tuples
+
+        Returns:
+            Slope in MB per minute
+
+        Algorithm:
+            Simple two-point slope: (y2 - y1) / (x2 - x1)
+            Where x is time in minutes, y is memory in MB
+        """
+        if len(measurements) < 2:
+            return 0.0
+
+        # Get first and last measurements
+        first_timestamp, first_memory = measurements[0]
+        last_timestamp, last_memory = measurements[-1]
+
+        # Calculate time delta in minutes
+        time_delta_seconds = (last_timestamp - first_timestamp).total_seconds()
+        time_delta_minutes = time_delta_seconds / 60.0
+
+        if time_delta_minutes == 0:
+            return 0.0
+
+        # Calculate slope (MB per minute)
+        memory_delta = last_memory - first_memory
+        return memory_delta / time_delta_minutes
+
+    def _trigger_leak_callbacks(self, deployment_id: str, trend: MemoryTrend) -> None:
+        """
+        Trigger registered callbacks for leak detection.
+
+        Args:
+            deployment_id: Deployment that has a leak
+            trend: MemoryTrend with leak analysis
+        """
+        for callback in self._leak_callbacks:
+            try:
+                callback(deployment_id, trend)
+            except Exception as e:
+                self.log_error(f"Error in leak callback {callback.__name__}: {e}")
+
+    def get_measurements(self, deployment_id: str) -> List[Tuple[datetime, float]]:
+        """
+        Get all measurements for a deployment (for testing/debugging).
+
+        Args:
+            deployment_id: Deployment identifier
+
+        Returns:
+            List of (timestamp, memory_mb) tuples
+        """
+        with self._lock:
+            return list(self._measurements.get(deployment_id, []))
+
+    def clear_measurements(self, deployment_id: str) -> None:
+        """
+        Clear measurements for a deployment (e.g., after restart).
+
+        Args:
+            deployment_id: Deployment identifier
+        """
+        with self._lock:
+            if deployment_id in self._measurements:
+                del self._measurements[deployment_id]
+                self.log_debug(f"Cleared measurements for {deployment_id}")
+
+
+__all__ = ["MemoryLeakDetector"]