claude-mpm 4.13.2__py3-none-any.whl → 4.14.0__py3-none-any.whl

claude_mpm/services/local_ops/health_checks/resource_check.py

@@ -0,0 +1,254 @@
+"""
+Resource Health Check for Claude MPM Framework
+===============================================
+
+WHY: Provides resource usage monitoring including CPU, memory, file descriptors,
+threads, and network connections to detect resource exhaustion issues.
+
+DESIGN DECISION: Uses psutil for cross-platform resource monitoring with
+configurable thresholds for different resource types.
+
+ARCHITECTURE:
+- CPU usage monitoring (threshold: 80%)
+- Memory usage monitoring (threshold: 500MB)
+- File descriptor count (threshold: 1000, Unix only)
+- Thread count monitoring
+- Network connection count (open sockets)
+
+USAGE:
+    resource_check = ResourceHealthCheck(process_manager)
+    result = resource_check.check(
+        deployment_id="my-app",
+        cpu_threshold=80.0,
+        memory_threshold_mb=500.0
+    )
+"""
+
+import platform
+
+import psutil
+
+from claude_mpm.services.core.base import SyncBaseService
+from claude_mpm.services.core.interfaces.health import IHealthCheck
+from claude_mpm.services.core.interfaces.process import ILocalProcessManager
+from claude_mpm.services.core.models.health import HealthCheckResult, HealthStatus
+
+
+class ResourceHealthCheck(SyncBaseService, IHealthCheck):
+    """
+    Resource usage health check implementation.
+
+    WHY: Monitors resource consumption to detect issues before they
+    cause service degradation or failures.
+
+    Thread Safety: Stateless, safe for concurrent execution.
+    """
+
+    # Default thresholds
+    DEFAULT_CPU_THRESHOLD = 80.0  # Percentage
+    DEFAULT_MEMORY_THRESHOLD_MB = 500.0  # Megabytes
+    DEFAULT_FD_THRESHOLD = 1000  # File descriptors (Unix only)
+    DEFAULT_THREAD_THRESHOLD = 100  # Threads
+
+    def __init__(self, process_manager: ILocalProcessManager):
+        """
+        Initialize resource health check.
+
+        Args:
+            process_manager: Process manager for deployment lookup
+        """
+        super().__init__("ResourceHealthCheck")
+        self.process_manager = process_manager
+        self.is_windows = platform.system() == "Windows"
+
+    def initialize(self) -> bool:
+        """
+        Initialize the health check.
+
+        Returns:
+            True if initialization successful
+        """
+        self._initialized = True
+        self.log_info("Resource health check initialized")
+        return True
+
+    def shutdown(self) -> None:
+        """Shutdown health check (no resources to clean up)."""
+        self._shutdown = True
+
+    def get_check_type(self) -> str:
+        """Get the check type identifier."""
+        return "resource"
+
+    def check(self, deployment_id: str, **kwargs) -> HealthCheckResult:
+        """
+        Execute resource health check for a deployment.
+
+        Args:
+            deployment_id: Unique deployment identifier
+            **kwargs: Optional parameters:
+                - cpu_threshold: CPU usage threshold percentage (default: 80.0)
+                - memory_threshold_mb: Memory usage threshold in MB (default: 500.0)
+                - fd_threshold: File descriptor threshold (default: 1000, Unix only)
+                - thread_threshold: Thread count threshold (default: 100)
+
+        Returns:
+            HealthCheckResult with check status and details
+
+        Raises:
+            ValueError: If deployment_id not found
+        """
+        # Validate deployment exists
+        deployment = self.process_manager.state_manager.get_deployment(deployment_id)
+        if not deployment:
+            raise ValueError(f"Deployment not found: {deployment_id}")
+
+        # Get thresholds from kwargs
+        cpu_threshold = kwargs.get("cpu_threshold", self.DEFAULT_CPU_THRESHOLD)
+        memory_threshold_mb = kwargs.get(
+            "memory_threshold_mb", self.DEFAULT_MEMORY_THRESHOLD_MB
+        )
+        fd_threshold = kwargs.get("fd_threshold", self.DEFAULT_FD_THRESHOLD)
+        thread_threshold = kwargs.get("thread_threshold", self.DEFAULT_THREAD_THRESHOLD)
+
+        try:
+            process = psutil.Process(deployment.process_id)
+
+            # Collect resource metrics
+            details = {
+                "pid": deployment.process_id,
+                "deployment_id": deployment_id,
+            }
+
+            issues = []
+
+            # Check CPU usage
+            try:
+                cpu_percent = process.cpu_percent(interval=0.1)
+                details["cpu_percent"] = round(cpu_percent, 2)
+                details["cpu_threshold"] = cpu_threshold
+
+                if cpu_percent > cpu_threshold:
+                    issues.append(
+                        f"High CPU usage: {cpu_percent:.1f}% (threshold: {cpu_threshold}%)"
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            # Check memory usage
+            try:
+                memory_info = process.memory_info()
+                memory_mb = memory_info.rss / (1024 * 1024)
+                details["memory_mb"] = round(memory_mb, 2)
+                details["memory_threshold_mb"] = memory_threshold_mb
+
+                if memory_mb > memory_threshold_mb:
+                    issues.append(
+                        f"High memory usage: {memory_mb:.1f}MB (threshold: {memory_threshold_mb}MB)"
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            # Check file descriptors (Unix only)
+            if not self.is_windows:
+                try:
+                    num_fds = process.num_fds()
+                    details["num_fds"] = num_fds
+                    details["fd_threshold"] = fd_threshold
+
+                    if num_fds > fd_threshold:
+                        issues.append(
+                            f"High file descriptor count: {num_fds} (threshold: {fd_threshold})"
+                        )
+                except (
+                    psutil.NoSuchProcess,
+                    psutil.AccessDenied,
+                    AttributeError,
+                ):
+                    # num_fds() not available on all platforms
+                    pass
+
+            # Check thread count
+            try:
+                num_threads = process.num_threads()
+                details["num_threads"] = num_threads
+                details["thread_threshold"] = thread_threshold
+
+                if num_threads > thread_threshold:
+                    issues.append(
+                        f"High thread count: {num_threads} (threshold: {thread_threshold})"
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            # Check connection count
+            try:
+                connections = process.net_connections()
+                num_connections = len(connections)
+                details["num_connections"] = num_connections
+
+                # Add connection breakdown by state
+                connection_states = {}
+                for conn in connections:
+                    state = conn.status
+                    connection_states[state] = connection_states.get(state, 0) + 1
+                details["connection_states"] = connection_states
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            # Determine health status based on issues
+            if issues:
+                return HealthCheckResult(
+                    status=HealthStatus.DEGRADED,
+                    check_type=self.get_check_type(),
+                    message=f"Resource usage issues detected: {'; '.join(issues)}",
+                    details=details,
+                )
+            return HealthCheckResult(
+                status=HealthStatus.HEALTHY,
+                check_type=self.get_check_type(),
+                message="Resource usage within normal limits",
+                details=details,
+            )
+
+        except psutil.NoSuchProcess:
+            # Process does not exist
+            return HealthCheckResult(
+                status=HealthStatus.UNHEALTHY,
+                check_type=self.get_check_type(),
+                message="Process no longer exists",
+                details={
+                    "pid": deployment.process_id,
+                    "deployment_id": deployment_id,
+                },
+            )
+
+        except psutil.AccessDenied as e:
+            # Cannot access process information
+            return HealthCheckResult(
+                status=HealthStatus.UNKNOWN,
+                check_type=self.get_check_type(),
+                message="Cannot access process resource information",
+                details={
+                    "pid": deployment.process_id,
+                    "deployment_id": deployment_id,
+                    "error": str(e),
+                },
+            )
+
+        except Exception as e:
+            # Unexpected error
+            self.log_error(f"Unexpected error in resource health check: {e}")
+            return HealthCheckResult(
+                status=HealthStatus.UNKNOWN,
+                check_type=self.get_check_type(),
+                message="Health check failed with error",
+                details={
+                    "pid": deployment.process_id,
+                    "deployment_id": deployment_id,
+                    "error": str(e),
+                },
+            )
+
+
+__all__ = ["ResourceHealthCheck"]

claude_mpm/services/local_ops/health_manager.py

@@ -0,0 +1,430 @@
+"""
+Health Check Manager for Claude MPM Framework
+==============================================
+
+WHY: Orchestrates multiple health check types, provides background monitoring,
+and maintains historical health data for local deployments.
+
+DESIGN DECISION: Uses background daemon thread for continuous monitoring with
+configurable check intervals. Aggregates results from all health check types
+using defined priority rules.
+
+ARCHITECTURE:
+- Orchestrates HTTP, process, and resource health checks
+- Background monitoring thread with configurable interval (default: 30s)
+- Thread-safe status tracking with threading.Lock
+- Historical health data (last 100 checks per deployment)
+- Health status aggregation with priority:
+  1. Process UNHEALTHY = Deployment UNHEALTHY (critical)
+  2. Any check UNHEALTHY = Deployment DEGRADED (service issues)
+  3. All checks HEALTHY = Deployment HEALTHY
+  4. Otherwise = UNKNOWN
+- Event callbacks for status changes
+
+USAGE:
+    health_manager = HealthCheckManager(
+        process_manager=process_manager,
+        check_interval=30,
+    )
+    health_manager.start_monitoring()
+
+    # Check health on-demand
+    health = health_manager.check_health(deployment_id)
+
+    # Stop monitoring
+    health_manager.stop_monitoring()
+"""
+
+import threading
+from collections import defaultdict
+from typing import Callable, Dict, List, Optional
+
+from claude_mpm.services.core.base import SyncBaseService
+from claude_mpm.services.core.interfaces.health import IHealthCheckManager
+from claude_mpm.services.core.interfaces.process import ILocalProcessManager
+from claude_mpm.services.core.models.health import (
+    DeploymentHealth,
+    HealthCheckResult,
+    HealthStatus,
+)
+from claude_mpm.services.local_ops.health_checks import (
+    HttpHealthCheck,
+    ProcessHealthCheck,
+    ResourceHealthCheck,
+)
+
+
+class HealthCheckManager(SyncBaseService, IHealthCheckManager):
+    """
+    Health check orchestration and monitoring service.
+
+    WHY: Provides comprehensive health monitoring by coordinating multiple
+    check types, maintaining historical data, and enabling background monitoring.
+
+    Thread Safety: All public methods are thread-safe with proper locking.
+    """
+
+    def __init__(
+        self,
+        process_manager: ILocalProcessManager,
+        check_interval: int = 30,
+        history_limit: int = 100,
+    ):
+        """
+        Initialize health check manager.
+
+        Args:
+            process_manager: Process manager for deployment lookup
+            check_interval: Background check interval in seconds (default: 30)
+            history_limit: Maximum historical entries per deployment (default: 100)
+        """
+        super().__init__("HealthCheckManager")
+        self.process_manager = process_manager
+        self.check_interval = check_interval
+        self.history_limit = history_limit
+
+        # Initialize health check implementations
+        self.http_check = HttpHealthCheck(process_manager)
+        self.process_check = ProcessHealthCheck(process_manager)
+        self.resource_check = ResourceHealthCheck(process_manager)
+
+        # Background monitoring state
+        self._monitoring = False
+        self._monitor_thread: Optional[threading.Thread] = None
+        self._stop_event = threading.Event()
+        self._lock = threading.Lock()
+
+        # Health history: deployment_id -> List[DeploymentHealth]
+        self._health_history: Dict[str, List[DeploymentHealth]] = defaultdict(list)
+
+        # Status change callbacks
+        self._status_callbacks: List[Callable] = []
+
+    def initialize(self) -> bool:
+        """
+        Initialize the health check manager.
+
+        Returns:
+            True if initialization successful
+        """
+        try:
+            # Initialize all health check implementations
+            if not self.http_check.initialize():
+                self.log_error("Failed to initialize HTTP health check")
+                return False
+
+            if not self.process_check.initialize():
+                self.log_error("Failed to initialize process health check")
+                return False
+
+            if not self.resource_check.initialize():
+                self.log_error("Failed to initialize resource health check")
+                return False
+
+            self._initialized = True
+            self.log_info("Health check manager initialized")
+            return True
+
+        except Exception as e:
+            self.log_error(f"Failed to initialize: {e}")
+            return False
+
+    def shutdown(self) -> None:
+        """Shutdown health check manager and stop monitoring."""
+        if self._monitoring:
+            self.stop_monitoring()
+
+        # Shutdown health check implementations
+        self.http_check.shutdown()
+        self.process_check.shutdown()
+        self.resource_check.shutdown()
+
+        self._shutdown = True
+        self.log_info("Health check manager shutdown complete")
+
+    def check_health(self, deployment_id: str, **kwargs) -> DeploymentHealth:
+        """
+        Execute all health checks for a deployment.
+
+        Args:
+            deployment_id: Unique deployment identifier
+            **kwargs: Optional parameters passed to health checks:
+                - endpoint: HTTP endpoint URL
+                - timeout: HTTP timeout in seconds
+                - cpu_threshold: CPU usage threshold percentage
+                - memory_threshold_mb: Memory usage threshold in MB
+
+        Returns:
+            DeploymentHealth with aggregated status and check results
+
+        Raises:
+            ValueError: If deployment_id not found
+        """
+        # Validate deployment exists
+        deployment = self.process_manager.state_manager.get_deployment(deployment_id)
+        if not deployment:
+            raise ValueError(f"Deployment not found: {deployment_id}")
+
+        # Execute all health checks
+        checks: List[HealthCheckResult] = []
+
+        # 1. Process health check (most critical)
+        try:
+            process_result = self.process_check.check(deployment_id, **kwargs)
+            checks.append(process_result)
+        except Exception as e:
+            self.log_error(f"Process health check failed: {e}")
+            checks.append(
+                HealthCheckResult(
+                    status=HealthStatus.UNKNOWN,
+                    check_type="process",
+                    message=f"Check failed: {e}",
+                    details={"error": str(e)},
+                )
+            )
+
+        # 2. Resource health check
+        try:
+            resource_result = self.resource_check.check(deployment_id, **kwargs)
+            checks.append(resource_result)
+        except Exception as e:
+            self.log_error(f"Resource health check failed: {e}")
+            checks.append(
+                HealthCheckResult(
+                    status=HealthStatus.UNKNOWN,
+                    check_type="resource",
+                    message=f"Check failed: {e}",
+                    details={"error": str(e)},
+                )
+            )
+
+        # 3. HTTP health check (optional, only if endpoint configured)
+        try:
+            http_result = self.http_check.check(deployment_id, **kwargs)
+            # Only add if check was actually performed (not UNKNOWN due to no endpoint)
+            if http_result.status != HealthStatus.UNKNOWN or kwargs.get("endpoint"):
+                checks.append(http_result)
+        except Exception as e:
+            self.log_error(f"HTTP health check failed: {e}")
+            checks.append(
+                HealthCheckResult(
+                    status=HealthStatus.UNKNOWN,
+                    check_type="http",
+                    message=f"Check failed: {e}",
+                    details={"error": str(e)},
+                )
+            )
+
+        # Aggregate health status
+        overall_status = self._aggregate_health_status(checks)
+
+        # Create deployment health
+        deployment_health = DeploymentHealth(
+            deployment_id=deployment_id,
+            overall_status=overall_status,
+            checks=checks,
+        )
+
+        # Update health history
+        with self._lock:
+            self._health_history[deployment_id].append(deployment_health)
+            # Trim history to limit
+            if len(self._health_history[deployment_id]) > self.history_limit:
+                self._health_history[deployment_id] = self._health_history[
+                    deployment_id
+                ][-self.history_limit :]
+
+            # Check for status changes and trigger callbacks
+            if len(self._health_history[deployment_id]) >= 2:
+                previous_health = self._health_history[deployment_id][-2]
+                if previous_health.overall_status != overall_status:
+                    self._trigger_status_callbacks(
+                        deployment_id, previous_health.overall_status, overall_status
+                    )
+
+        return deployment_health
+
+    def start_monitoring(self) -> None:
+        """
+        Start background health monitoring.
+
+        WHY: Enables continuous health tracking without manual polling.
+        Creates a daemon thread that performs periodic checks.
+        """
+        with self._lock:
+            if self._monitoring:
+                self.log_warning("Health monitoring already running")
+                return
+
+            self._monitoring = True
+            self._stop_event.clear()
+
+            # Create and start monitoring thread
+            self._monitor_thread = threading.Thread(
+                target=self._monitor_loop, daemon=True, name="HealthMonitorThread"
+            )
+            self._monitor_thread.start()
+
+            self.log_info(
+                f"Started health monitoring with {self.check_interval}s interval"
+            )
+
+    def stop_monitoring(self) -> None:
+        """
+        Stop background health monitoring.
+
+        WHY: Gracefully stops the monitoring thread and releases resources.
+        """
+        with self._lock:
+            if not self._monitoring:
+                return
+
+            self._monitoring = False
+            self._stop_event.set()
+
+        # Wait for monitoring thread to stop
+        if self._monitor_thread and self._monitor_thread.is_alive():
+            self._monitor_thread.join(timeout=5.0)
+
+        self.log_info("Stopped health monitoring")
+
+    def is_monitoring(self) -> bool:
+        """
+        Check if background monitoring is active.
+
+        Returns:
+            True if monitoring thread is running
+        """
+        with self._lock:
+            return self._monitoring
+
+    def get_health_history(
+        self, deployment_id: str, limit: int = 10
+    ) -> List[DeploymentHealth]:
+        """
+        Get historical health check results for a deployment.
+
+        Args:
+            deployment_id: Unique deployment identifier
+            limit: Maximum number of historical entries to return
+
+        Returns:
+            List of DeploymentHealth objects, newest first
+        """
+        with self._lock:
+            history = self._health_history.get(deployment_id, [])
+            return list(reversed(history[-limit:]))
+
+    def register_status_callback(
+        self, callback: Callable[[str, HealthStatus, HealthStatus], None]
+    ) -> None:
+        """
+        Register a callback for health status changes.
+
+        Args:
+            callback: Function called with (deployment_id, old_status, new_status)
+        """
+        with self._lock:
+            self._status_callbacks.append(callback)
+            self.log_debug(f"Registered status callback: {callback.__name__}")
+
+    def _monitor_loop(self) -> None:
+        """
+        Background monitoring loop.
+
+        WHY: Runs in a separate thread to perform periodic health checks
+        on all active deployments.
+        """
+        self.log_debug("Health monitoring loop started")
+
+        while not self._stop_event.is_set():
+            try:
+                # Get all active deployments
+                deployments = self.process_manager.state_manager.get_all_deployments()
+
+                for deployment in deployments:
+                    if self._stop_event.is_set():
+                        break
+
+                    try:
+                        # Perform health check
+                        self.check_health(deployment.deployment_id)
+                    except Exception as e:
+                        self.log_error(
+                            f"Error checking health for {deployment.deployment_id}: {e}"
+                        )
+
+                # Sleep until next check interval
+                # Use wait() instead of sleep() for faster shutdown response
+                self._stop_event.wait(timeout=self.check_interval)
+
+            except Exception as e:
+                self.log_error(f"Error in health monitoring loop: {e}")
+                # Don't crash the thread, just continue
+                self._stop_event.wait(timeout=1.0)
+
+        self.log_debug("Health monitoring loop stopped")
+
+    def _aggregate_health_status(self, checks: List[HealthCheckResult]) -> HealthStatus:
+        """
+        Aggregate health status from multiple check results.
+
+        WHY: Combines results from different check types using priority rules
+        to determine overall deployment health.
+
+        Priority Rules:
+        1. Process UNHEALTHY = Deployment UNHEALTHY (critical)
+        2. Any check UNHEALTHY = Deployment DEGRADED (service issues but process alive)
+        3. All checks HEALTHY = Deployment HEALTHY
+        4. Otherwise = UNKNOWN
+
+        Args:
+            checks: List of health check results
+
+        Returns:
+            Aggregated HealthStatus
+        """
+        if not checks:
+            return HealthStatus.UNKNOWN
+
+        # Get process check result (most critical)
+        process_check = next((c for c in checks if c.check_type == "process"), None)
+
+        # Rule 1: Process UNHEALTHY = Deployment UNHEALTHY
+        if process_check and process_check.status == HealthStatus.UNHEALTHY:
+            return HealthStatus.UNHEALTHY
+
+        # Rule 2: Any check UNHEALTHY (but process alive) = DEGRADED
+        if any(c.status == HealthStatus.UNHEALTHY for c in checks):
+            return HealthStatus.DEGRADED
+
+        # Check for degraded status
+        if any(c.status == HealthStatus.DEGRADED for c in checks):
+            return HealthStatus.DEGRADED
+
+        # Rule 3: All checks HEALTHY = Deployment HEALTHY
+        if all(c.status == HealthStatus.HEALTHY for c in checks):
+            return HealthStatus.HEALTHY
+
+        # Rule 4: Otherwise = UNKNOWN
+        return HealthStatus.UNKNOWN
+
+    def _trigger_status_callbacks(
+        self, deployment_id: str, old_status: HealthStatus, new_status: HealthStatus
+    ) -> None:
+        """
+        Trigger registered callbacks for status changes.
+
+        Args:
+            deployment_id: Deployment that changed status
+            old_status: Previous health status
+            new_status: New health status
+        """
+        for callback in self._status_callbacks:
+            try:
+                callback(deployment_id, old_status, new_status)
+            except Exception as e:
+                self.log_error(f"Error in status callback {callback.__name__}: {e}")
+
+
+__all__ = ["HealthCheckManager"]