claude-mpm 4.13.1__py3-none-any.whl → 4.14.0__py3-none-any.whl

claude_mpm/services/core/models/stability.py

@@ -0,0 +1,264 @@
+"""
+Stability Monitoring Data Models for Claude MPM Framework
+===========================================================
+
+WHY: This module defines data structures for stability monitoring operations,
+including memory leak detection, log pattern matching, and resource usage tracking.
+
+DESIGN DECISION: Uses dataclasses for immutability and type safety. Provides
+clear data structures for proactive monitoring and crash prevention.
+
+ARCHITECTURE:
+- MemoryTrend: Memory usage trend analysis with leak detection
+- LogPatternMatch: Log pattern match with severity and context
+- ResourceUsage: Comprehensive resource usage snapshot
+"""
+
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List
+
+
+@dataclass
+class MemoryTrend:
+    """
+    Memory usage trend analysis result.
+
+    WHY: Provides structured data for memory leak detection, including
+    historical measurements, slope calculation, and leak detection status.
+
+    Attributes:
+        deployment_id: Unique deployment identifier
+        timestamps: List of measurement timestamps
+        memory_mb: List of memory measurements in megabytes
+        slope_mb_per_minute: Calculated memory growth rate (MB/minute)
+        is_leaking: Whether a memory leak was detected
+        window_size: Number of measurements in the analysis window
+        threshold_mb_per_minute: Leak detection threshold used
+    """
+
+    deployment_id: str
+    timestamps: List[datetime] = field(default_factory=list)
+    memory_mb: List[float] = field(default_factory=list)
+    slope_mb_per_minute: float = 0.0
+    is_leaking: bool = False
+    window_size: int = 0
+    threshold_mb_per_minute: float = 10.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert to dictionary for JSON serialization.
+
+        Returns:
+            Dictionary representation with datetimes converted to ISO format
+        """
+        data = asdict(self)
+        data["timestamps"] = [ts.isoformat() for ts in self.timestamps]
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "MemoryTrend":
+        """
+        Create MemoryTrend from dictionary.
+
+        Args:
+            data: Dictionary from JSON deserialization
+
+        Returns:
+            MemoryTrend instance
+        """
+        # Convert ISO strings to datetime
+        if isinstance(data.get("timestamps"), list):
+            data["timestamps"] = [
+                datetime.fromisoformat(ts) if isinstance(ts, str) else ts
+                for ts in data["timestamps"]
+            ]
+
+        return cls(**data)
+
+    @property
+    def latest_memory_mb(self) -> float:
+        """Get the most recent memory measurement."""
+        return self.memory_mb[-1] if self.memory_mb else 0.0
+
+    @property
+    def oldest_memory_mb(self) -> float:
+        """Get the oldest memory measurement in the window."""
+        return self.memory_mb[0] if self.memory_mb else 0.0
+
+    @property
+    def time_span_minutes(self) -> float:
+        """Get the time span covered by the measurements in minutes."""
+        if len(self.timestamps) < 2:
+            return 0.0
+        delta = self.timestamps[-1] - self.timestamps[0]
+        return delta.total_seconds() / 60.0
+
+
+@dataclass
+class LogPatternMatch:
+    """
+    Result of a log pattern match.
+
+    WHY: Contains all information about a detected error pattern in logs,
+    enabling analysis, alerting, and debugging of issues before they cause crashes.
+
+    Attributes:
+        deployment_id: Unique deployment identifier
+        pattern: Regex pattern that matched
+        line: The log line that matched
+        timestamp: When the match was detected
+        severity: Error severity level (ERROR, CRITICAL, WARNING)
+        line_number: Line number in log file (if available)
+        context: Additional context lines (before/after)
+    """
+
+    deployment_id: str
+    pattern: str
+    line: str
+    timestamp: datetime = field(default_factory=datetime.now)
+    severity: str = "ERROR"
+    line_number: int = 0
+    context: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert to dictionary for JSON serialization.
+
+        Returns:
+            Dictionary representation with datetime converted to ISO format
+        """
+        data = asdict(self)
+        data["timestamp"] = self.timestamp.isoformat()
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "LogPatternMatch":
+        """
+        Create LogPatternMatch from dictionary.
+
+        Args:
+            data: Dictionary from JSON deserialization
+
+        Returns:
+            LogPatternMatch instance
+        """
+        # Convert ISO string to datetime
+        if isinstance(data.get("timestamp"), str):
+            data["timestamp"] = datetime.fromisoformat(data["timestamp"])
+
+        return cls(**data)
+
+    @property
+    def is_critical(self) -> bool:
+        """Check if this match represents a critical error."""
+        return self.severity == "CRITICAL"
+
+
+@dataclass
+class ResourceUsage:
+    """
+    Comprehensive resource usage snapshot.
+
+    WHY: Provides detailed resource consumption metrics across multiple
+    resource types to enable preemptive action before exhaustion.
+
+    Attributes:
+        deployment_id: Unique deployment identifier
+        file_descriptors: Current file descriptor count
+        max_file_descriptors: Maximum file descriptors allowed (ulimit -n)
+        threads: Current thread count
+        connections: Current network connection count
+        disk_free_mb: Free disk space in working directory (MB)
+        is_critical: Whether any resource exceeds 80% threshold
+        timestamp: When the measurement was taken
+        details: Additional resource-specific details
+    """
+
+    deployment_id: str
+    file_descriptors: int = 0
+    max_file_descriptors: int = 0
+    threads: int = 0
+    connections: int = 0
+    disk_free_mb: float = 0.0
+    is_critical: bool = False
+    timestamp: datetime = field(default_factory=datetime.now)
+    details: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert to dictionary for JSON serialization.
+
+        Returns:
+            Dictionary representation with datetime converted to ISO format
+        """
+        data = asdict(self)
+        data["timestamp"] = self.timestamp.isoformat()
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ResourceUsage":
+        """
+        Create ResourceUsage from dictionary.
+
+        Args:
+            data: Dictionary from JSON deserialization
+
+        Returns:
+            ResourceUsage instance
+        """
+        # Convert ISO string to datetime
+        if isinstance(data.get("timestamp"), str):
+            data["timestamp"] = datetime.fromisoformat(data["timestamp"])
+
+        return cls(**data)
+
+    @property
+    def fd_usage_percent(self) -> float:
+        """Calculate file descriptor usage percentage."""
+        if self.max_file_descriptors == 0:
+            return 0.0
+        return (self.file_descriptors / self.max_file_descriptors) * 100.0
+
+    @property
+    def is_fd_critical(self) -> bool:
+        """Check if file descriptor usage is critical (>80%)."""
+        return self.fd_usage_percent >= 80.0  # >= instead of > for 80% exactly
+
+    def get_critical_resources(self) -> List[str]:
+        """
+        Get list of resources at critical levels.
+
+        Returns:
+            List of resource names exceeding 80% threshold
+        """
+        critical = []
+
+        if self.is_fd_critical:
+            critical.append(
+                f"file_descriptors ({self.file_descriptors}/{self.max_file_descriptors})"
+            )
+
+        # Check thread count (threshold from details if available)
+        thread_threshold = self.details.get("thread_threshold", 1000)
+        if self.threads > thread_threshold * 0.8:
+            critical.append(f"threads ({self.threads})")
+
+        # Check connection count (threshold from details if available)
+        connection_threshold = self.details.get("connection_threshold", 500)
+        if self.connections > connection_threshold * 0.8:
+            critical.append(f"connections ({self.connections})")
+
+        # Check disk space (threshold from details if available)
+        disk_threshold_mb = self.details.get("disk_threshold_mb", 100)
+        if self.disk_free_mb < disk_threshold_mb:
+            critical.append(f"disk_space ({self.disk_free_mb:.1f}MB free)")
+
+        return critical
+
+
+__all__ = [
+    "LogPatternMatch",
+    "MemoryTrend",
+    "ResourceUsage",
+]

claude_mpm/services/local_ops/__init__.py

@@ -0,0 +1,163 @@
+"""
+Local Operations Service Package
+=================================
+
+WHY: Provides process management and health monitoring capabilities for local
+development deployments. This package implements the core infrastructure needed
+by the local-ops-agent to spawn, track, manage, and monitor background processes.
+
+DESIGN DECISION: Organized as a service package under services/ to integrate
+with the existing service architecture and dependency injection system.
+
+ARCHITECTURE:
+- StateManager: Persistent deployment state tracking
+- ProcessManager: Process lifecycle management with isolation
+- HealthCheckManager: Three-tier health monitoring (HTTP, process, resource)
+- HealthChecks: HTTP, process, and resource health check implementations
+- CrashDetector: Crash detection via health status monitoring
+- RestartPolicy: Intelligent restart policies with exponential backoff
+- RestartManager: Auto-restart orchestration with circuit breaker
+- MemoryLeakDetector: Proactive memory leak detection using trend analysis
+- LogMonitor: Real-time log file monitoring for error patterns
+- ResourceMonitor: Comprehensive resource exhaustion prevention
+- Data Models: Process status, deployment state, health status, restart history,
+              stability metrics (memory trends, log matches, resource usage)
+- Interfaces: ILocalProcessManager, IDeploymentStateManager, IHealthCheckManager,
+             ICrashDetector, IRestartPolicy, IRestartManager, IMemoryLeakDetector,
+             ILogMonitor, IResourceMonitor
+
+USAGE:
+    from claude_mpm.services.local_ops import (
+        LocalProcessManager,
+        DeploymentStateManager,
+        HealthCheckManager,
+        StartConfig,
+        ProcessStatus,
+        HealthStatus,
+    )
+
+    # Initialize managers
+    state_manager = DeploymentStateManager(".claude-mpm/deployment-state.json")
+    process_manager = LocalProcessManager(state_manager)
+    health_manager = HealthCheckManager(process_manager, check_interval=30)
+
+    # Start a process
+    config = StartConfig(
+        command=["npm", "run", "dev"],
+        working_directory="/path/to/project",
+        port=3000
+    )
+    deployment = process_manager.start(config)
+
+    # Check health
+    health = health_manager.check_health(deployment.deployment_id)
+
+    # Start background monitoring
+    health_manager.start_monitoring()
+"""
+
+# Re-export data models and interfaces for convenience
+from claude_mpm.services.core.interfaces.health import (
+    IHealthCheck,
+    IHealthCheckManager,
+)
+from claude_mpm.services.core.interfaces.process import (
+    IDeploymentStateManager,
+    ILocalProcessManager,
+)
+from claude_mpm.services.core.interfaces.restart import (
+    ICrashDetector,
+    IRestartManager,
+    IRestartPolicy,
+)
+from claude_mpm.services.core.interfaces.stability import (
+    ILogMonitor,
+    IMemoryLeakDetector,
+    IResourceMonitor,
+)
+from claude_mpm.services.core.models.health import (
+    DeploymentHealth,
+    HealthCheckResult,
+    HealthStatus,
+)
+from claude_mpm.services.core.models.process import (
+    PROTECTED_PORT_RANGES,
+    DeploymentState,
+    ProcessInfo,
+    ProcessStatus,
+    StartConfig,
+    is_port_protected,
+)
+from claude_mpm.services.core.models.restart import (
+    CircuitBreakerState,
+    RestartAttempt,
+    RestartConfig,
+    RestartHistory,
+)
+from claude_mpm.services.core.models.stability import (
+    LogPatternMatch,
+    MemoryTrend,
+    ResourceUsage,
+)
+
+# Import service implementations
+from .health_manager import HealthCheckManager
+from .log_monitor import LogMonitor
+from .memory_leak_detector import MemoryLeakDetector
+from .process_manager import (
+    LocalProcessManager,
+    PortConflictError,
+    ProcessSpawnError,
+)
+from .resource_monitor import ResourceMonitor
+from .state_manager import DeploymentStateManager, StateCorruptionError
+from .unified_manager import UnifiedLocalOpsManager
+
+__all__ = [
+    "PROTECTED_PORT_RANGES",
+    # Data models - Restart
+    "CircuitBreakerState",
+    "CrashDetector",
+    "DeploymentHealth",
+    "DeploymentState",
+    # Service implementations
+    "DeploymentStateManager",
+    "HealthCheckManager",
+    "HealthCheckResult",
+    # Data models - Health
+    "HealthStatus",
+    "ICrashDetector",
+    # Interfaces
+    "IDeploymentStateManager",
+    "IHealthCheck",
+    "IHealthCheckManager",
+    "ILocalProcessManager",
+    "ILogMonitor",
+    "IMemoryLeakDetector",
+    "IResourceMonitor",
+    "IRestartManager",
+    "IRestartPolicy",
+    "LocalProcessManager",
+    "LogMonitor",
+    "LogPatternMatch",
+    "MemoryLeakDetector",
+    # Data models - Stability
+    "MemoryTrend",
+    "PortConflictError",
+    "ProcessInfo",
+    "ProcessSpawnError",
+    # Data models - Process
+    "ProcessStatus",
+    "ResourceMonitor",
+    "ResourceUsage",
+    "RestartAttempt",
+    "RestartConfig",
+    "RestartHistory",
+    "RestartManager",
+    "RestartPolicy",
+    "StartConfig",
+    # Exceptions
+    "StateCorruptionError",
+    "UnifiedLocalOpsManager",
+    "is_port_protected",
+]

claude_mpm/services/local_ops/crash_detector.py

@@ -0,0 +1,257 @@
+"""
+Crash Detector for Claude MPM Framework
+========================================
+
+WHY: Detects process crashes and failures by monitoring health status changes,
+process exits, and zombie states. Integrates with HealthCheckManager to receive
+real-time status updates.
+
+DESIGN DECISION: Uses callback-based architecture to receive health status
+changes from HealthCheckManager. Tracks crash history per deployment to
+enable pattern detection and intelligent restart policies.
+
+ARCHITECTURE:
+- Subscribes to HealthCheckManager status change callbacks
+- Detects crashes when status transitions to UNHEALTHY
+- Tracks crash count per deployment
+- Invokes registered crash callbacks when crash detected
+
+USAGE:
+    crash_detector = CrashDetector(health_manager)
+    crash_detector.register_crash_callback(handle_crash)
+    crash_detector.start_monitoring(deployment_id)
+"""
+
+import threading
+from collections import defaultdict
+from typing import Callable, Dict, List, Set
+
+from claude_mpm.services.core.base import SyncBaseService
+from claude_mpm.services.core.interfaces.health import IHealthCheckManager
+from claude_mpm.services.core.interfaces.restart import ICrashDetector
+from claude_mpm.services.core.models.health import HealthStatus
+
+
+class CrashDetector(SyncBaseService, ICrashDetector):
+    """
+    Detects process crashes via health status monitoring.
+
+    WHY: Provides automated crash detection by monitoring health status
+    changes. Enables reactive restart policies based on crash events.
+
+    Thread Safety: All public methods are thread-safe with proper locking.
+    """
+
+    def __init__(self, health_manager: IHealthCheckManager):
+        """
+        Initialize crash detector.
+
+        Args:
+            health_manager: Health check manager for status monitoring
+        """
+        super().__init__("CrashDetector")
+        self.health_manager = health_manager
+        self._lock = threading.Lock()
+
+        # Deployments being monitored
+        self._monitored_deployments: Set[str] = set()
+
+        # Last known health status per deployment
+        self._last_health_status: Dict[str, HealthStatus] = {}
+
+        # Crash count per deployment
+        self._crash_count: Dict[str, int] = defaultdict(int)
+
+        # Crash callbacks: List of functions called with (deployment_id, reason)
+        self._crash_callbacks: List[Callable[[str, str], None]] = []
+
+    def initialize(self) -> bool:
+        """
+        Initialize the crash detector.
+
+        Returns:
+            True if initialization successful
+        """
+        self.logger.info("Initializing CrashDetector")
+
+        # Register with health manager to receive status change callbacks
+        self.health_manager.register_status_callback(self._on_health_status_change)
+
+        self.logger.info("CrashDetector initialized successfully")
+        return True
+
+    def register_crash_callback(self, callback: Callable[[str, str], None]) -> None:
+        """
+        Register a callback to be invoked when a crash is detected.
+
+        Args:
+            callback: Function called with (deployment_id, reason)
+        """
+        with self._lock:
+            self._crash_callbacks.append(callback)
+            callback_name = getattr(callback, "__name__", repr(callback))
+            self.logger.debug(f"Registered crash callback: {callback_name}")
+
+    def start_monitoring(self, deployment_id: str) -> None:
+        """
+        Start monitoring a deployment for crashes.
+
+        Args:
+            deployment_id: Unique deployment identifier
+
+        Raises:
+            ValueError: If deployment_id not found
+        """
+        with self._lock:
+            self._monitored_deployments.add(deployment_id)
+            self.logger.info(
+                f"Started crash monitoring for deployment: {deployment_id}"
+            )
+
+            # Get initial health status
+            try:
+                health = self.health_manager.check_health(deployment_id)
+                self._last_health_status[deployment_id] = health.overall_status
+                self.logger.debug(
+                    f"Initial health status for {deployment_id}: {health.overall_status.value}"
+                )
+            except Exception as e:
+                self.logger.warning(
+                    f"Failed to get initial health status for {deployment_id}: {e}"
+                )
+                self._last_health_status[deployment_id] = HealthStatus.UNKNOWN
+
+    def stop_monitoring(self, deployment_id: str) -> None:
+        """
+        Stop monitoring a deployment.
+
+        Args:
+            deployment_id: Unique deployment identifier
+        """
+        with self._lock:
+            self._monitored_deployments.discard(deployment_id)
+            self._last_health_status.pop(deployment_id, None)
+            self.logger.info(
+                f"Stopped crash monitoring for deployment: {deployment_id}"
+            )
+
+    def is_monitoring(self, deployment_id: str) -> bool:
+        """
+        Check if a deployment is being monitored.
+
+        Args:
+            deployment_id: Unique deployment identifier
+
+        Returns:
+            True if deployment is being monitored
+        """
+        with self._lock:
+            return deployment_id in self._monitored_deployments
+
+    def get_crash_count(self, deployment_id: str) -> int:
+        """
+        Get the number of crashes detected for a deployment.
+
+        Args:
+            deployment_id: Unique deployment identifier
+
+        Returns:
+            Number of crashes detected
+        """
+        with self._lock:
+            return self._crash_count.get(deployment_id, 0)
+
+    def reset_crash_count(self, deployment_id: str) -> None:
+        """
+        Reset crash count for a deployment.
+
+        WHY: Allows manual intervention to clear crash history.
+
+        Args:
+            deployment_id: Unique deployment identifier
+        """
+        with self._lock:
+            self._crash_count[deployment_id] = 0
+            self.logger.debug(f"Reset crash count for deployment: {deployment_id}")
+
+    def shutdown(self) -> bool:
+        """
+        Shutdown the crash detector.
+
+        Returns:
+            True if shutdown successful
+        """
+        with self._lock:
+            self._monitored_deployments.clear()
+            self._last_health_status.clear()
+            self._crash_count.clear()
+            self._crash_callbacks.clear()
+        self.logger.info("CrashDetector shutdown successfully")
+        return True
+
+    def _on_health_status_change(
+        self, deployment_id: str, old_status: HealthStatus, new_status: HealthStatus
+    ) -> None:
+        """
+        Handle health status changes from HealthCheckManager.
+
+        WHY: Callback invoked by HealthCheckManager when status changes.
+        Detects crashes when status transitions to UNHEALTHY.
+
+        Args:
+            deployment_id: Unique deployment identifier
+            old_status: Previous health status
+            new_status: New health status
+        """
+        with self._lock:
+            # Only process if we're monitoring this deployment
+            if deployment_id not in self._monitored_deployments:
+                return
+
+            # Update last known status
+            self._last_health_status[deployment_id] = new_status
+
+            # Detect crash: transition from operational to UNHEALTHY
+            if old_status.is_operational() and new_status.is_critical():
+                self._handle_crash(
+                    deployment_id, "Health status transitioned to UNHEALTHY"
+                )
+
+            # Also detect: transition from UNKNOWN to UNHEALTHY (process died)
+            elif (
+                old_status == HealthStatus.UNKNOWN
+                and new_status == HealthStatus.UNHEALTHY
+            ):
+                self._handle_crash(deployment_id, "Process became unhealthy")
+
+    def _handle_crash(self, deployment_id: str, reason: str) -> None:
+        """
+        Handle detected crash.
+
+        WHY: Increments crash count and invokes all registered callbacks.
+
+        Args:
+            deployment_id: Unique deployment identifier
+            reason: Reason for crash detection
+        """
+        # Increment crash count
+        self._crash_count[deployment_id] += 1
+        crash_count = self._crash_count[deployment_id]
+
+        self.logger.warning(
+            f"Crash detected for deployment {deployment_id} "
+            f"(count: {crash_count}): {reason}"
+        )
+
+        # Invoke all crash callbacks
+        for callback in self._crash_callbacks:
+            try:
+                callback(deployment_id, reason)
+            except Exception as e:
+                self.logger.error(
+                    f"Error invoking crash callback {callback.__name__}: {e}",
+                    exc_info=True,
+                )
+
+
+__all__ = ["CrashDetector"]