unrealon 1.1.6__py3-none-any.whl → 2.0.5__py3-none-any.whl

unrealon_core/monitoring/health_check.py

@@ -0,0 +1,354 @@
+"""
+Health Check System
+
+Comprehensive health monitoring for all system components.
+Following critical requirements - max 500 lines, functions < 20 lines.
+
+Phase 2: Core Systems - Monitoring
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Dict, Any, List, Optional, Callable
+from enum import Enum
+
+from pydantic import BaseModel, Field, ConfigDict
+from ..utils.time import utc_now
+
+logger = logging.getLogger(__name__)
+
+
+class HealthStatus(str, Enum):
+    """Health status levels."""
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    UNKNOWN = "unknown"
+
+
+class HealthCheckResult(BaseModel):
+    """Result of a health check."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    component: str = Field(description="Component name")
+    status: HealthStatus = Field(description="Health status")
+    message: str = Field(description="Status message")
+    timestamp: datetime = Field(description="Check timestamp")
+    duration_ms: float = Field(description="Check duration in milliseconds")
+    details: Dict[str, Any] = Field(default_factory=dict, description="Additional details")
+    
+    def is_healthy(self) -> bool:
+        """Check if component is healthy."""
+        return self.status == HealthStatus.HEALTHY
+    
+    def is_degraded(self) -> bool:
+        """Check if component is degraded."""
+        return self.status == HealthStatus.DEGRADED
+
+
+class ComponentHealth(BaseModel):
+    """Health information for a component."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    name: str = Field(description="Component name")
+    current_status: HealthStatus = Field(description="Current health status")
+    last_check: datetime = Field(description="Last health check time")
+    check_count: int = Field(default=0, description="Total health checks performed")
+    failure_count: int = Field(default=0, description="Number of failed checks")
+    last_failure: Optional[datetime] = Field(default=None, description="Last failure time")
+    uptime_start: datetime = Field(description="When component became healthy")
+    
+    def get_uptime_seconds(self) -> float:
+        """Get uptime in seconds."""
+        if self.current_status != HealthStatus.HEALTHY:
+            return 0.0
+        return (utc_now() - self.uptime_start).total_seconds()
+    
+    def get_failure_rate(self) -> float:
+        """Get failure rate as percentage."""
+        if self.check_count == 0:
+            return 0.0
+        return (self.failure_count / self.check_count) * 100.0
+
+
+class HealthChecker:
+    """
+    Health checker for system components.
+    
+    Provides centralized health monitoring with configurable
+    check intervals and failure thresholds.
+    """
+    
+    def __init__(self, check_interval: float = 30.0):
+        """Initialize health checker."""
+        self.check_interval = check_interval
+        self._components: Dict[str, ComponentHealth] = {}
+        self._check_functions: Dict[str, Callable] = {}
+        self._running = False
+        self._check_task: Optional[asyncio.Task] = None
+        self.logger = logging.getLogger("health_checker")
+    
+    def register_component(
+        self, 
+        name: str, 
+        check_func: Callable[[], Any]
+    ) -> None:
+        """
+        Register component for health monitoring.
+        
+        Args:
+            name: Component name
+            check_func: Async function that returns health status
+        """
+        self._check_functions[name] = check_func
+        self._components[name] = ComponentHealth(
+            name=name,
+            current_status=HealthStatus.UNKNOWN,
+            last_check=utc_now(),
+            uptime_start=utc_now()
+        )
+        
+        self.logger.info(f"Registered health check for component: {name}")
+    
+    async def check_component(self, name: str) -> HealthCheckResult:
+        """
+        Perform health check for specific component.
+        
+        Args:
+            name: Component name
+            
+        Returns:
+            HealthCheckResult with check outcome
+        """
+        if name not in self._check_functions:
+            return HealthCheckResult(
+                component=name,
+                status=HealthStatus.UNKNOWN,
+                message="Component not registered",
+                timestamp=utc_now(),
+                duration_ms=0.0
+            )
+        
+        start_time = utc_now()
+        
+        try:
+            check_func = self._check_functions[name]
+            
+            # Execute health check with timeout
+            result = await asyncio.wait_for(check_func(), timeout=10.0)
+            
+            duration = (utc_now() - start_time).total_seconds() * 1000
+            
+            # Determine status from result
+            if isinstance(result, bool):
+                status = HealthStatus.HEALTHY if result else HealthStatus.UNHEALTHY
+                message = "Health check passed" if result else "Health check failed"
+                details = {}
+            elif isinstance(result, dict):
+                status = HealthStatus(result.get('status', 'unknown'))
+                message = result.get('message', 'No message')
+                details = result.get('details', {})
+            else:
+                status = HealthStatus.HEALTHY
+                message = str(result)
+                details = {}
+            
+            # Update component health
+            await self._update_component_health(name, status)
+            
+            return HealthCheckResult(
+                component=name,
+                status=status,
+                message=message,
+                timestamp=utc_now(),
+                duration_ms=duration,
+                details=details
+            )
+            
+        except asyncio.TimeoutError:
+            duration = (utc_now() - start_time).total_seconds() * 1000
+            await self._update_component_health(name, HealthStatus.UNHEALTHY)
+            
+            return HealthCheckResult(
+                component=name,
+                status=HealthStatus.UNHEALTHY,
+                message="Health check timed out",
+                timestamp=utc_now(),
+                duration_ms=duration
+            )
+            
+        except Exception as e:
+            duration = (utc_now() - start_time).total_seconds() * 1000
+            await self._update_component_health(name, HealthStatus.UNHEALTHY)
+            
+            return HealthCheckResult(
+                component=name,
+                status=HealthStatus.UNHEALTHY,
+                message=f"Health check error: {str(e)}",
+                timestamp=utc_now(),
+                duration_ms=duration
+            )
+    
+    async def check_all_components(self) -> List[HealthCheckResult]:
+        """Check health of all registered components."""
+        results = []
+        
+        for name in self._check_functions.keys():
+            result = await self.check_component(name)
+            results.append(result)
+        
+        return results
+    
+    async def _update_component_health(
+        self, 
+        name: str, 
+        status: HealthStatus
+    ) -> None:
+        """Update component health tracking."""
+        if name not in self._components:
+            return
+        
+        component = self._components[name]
+        component.last_check = utc_now()
+        component.check_count += 1
+        
+        # Track status changes
+        if status != component.current_status:
+            if status == HealthStatus.HEALTHY:
+                component.uptime_start = utc_now()
+                self.logger.info(f"Component {name} became healthy")
+            else:
+                self.logger.warning(f"Component {name} status changed to {status}")
+        
+        # Track failures
+        if status in [HealthStatus.UNHEALTHY, HealthStatus.DEGRADED]:
+            component.failure_count += 1
+            component.last_failure = utc_now()
+        
+        component.current_status = status
+    
+    async def start_monitoring(self) -> None:
+        """Start continuous health monitoring."""
+        if self._running:
+            self.logger.warning("Health monitoring already running")
+            return
+        
+        self._running = True
+        self._check_task = asyncio.create_task(self._monitoring_loop())
+        
+        self.logger.info(f"Started health monitoring (interval: {self.check_interval}s)")
+    
+    async def stop_monitoring(self) -> None:
+        """Stop health monitoring."""
+        self._running = False
+        
+        if self._check_task and not self._check_task.done():
+            self._check_task.cancel()
+            try:
+                await self._check_task
+            except asyncio.CancelledError:
+                pass
+        
+        self.logger.info("Stopped health monitoring")
+    
+    async def _monitoring_loop(self) -> None:
+        """Background monitoring loop."""
+        try:
+            while self._running:
+                await self.check_all_components()
+                await asyncio.sleep(self.check_interval)
+                
+        except asyncio.CancelledError:
+            self.logger.debug("Monitoring loop cancelled")
+            raise
+        except Exception as e:
+            self.logger.error(f"Error in monitoring loop: {e}")
+    
+    def get_system_health(self) -> Dict[str, Any]:
+        """Get overall system health summary."""
+        if not self._components:
+            return {
+                'overall_status': HealthStatus.UNKNOWN,
+                'components': {},
+                'summary': {
+                    'total': 0,
+                    'healthy': 0,
+                    'degraded': 0,
+                    'unhealthy': 0,
+                    'unknown': 0
+                }
+            }
+        
+        # Count component statuses
+        status_counts = {
+            HealthStatus.HEALTHY: 0,
+            HealthStatus.DEGRADED: 0,
+            HealthStatus.UNHEALTHY: 0,
+            HealthStatus.UNKNOWN: 0
+        }
+        
+        components_data = {}
+        
+        for name, component in self._components.items():
+            status_counts[component.current_status] += 1
+            components_data[name] = {
+                'status': component.current_status,
+                'last_check': component.last_check.isoformat(),
+                'uptime_seconds': component.get_uptime_seconds(),
+                'failure_rate': component.get_failure_rate(),
+                'check_count': component.check_count
+            }
+        
+        # Determine overall status
+        if status_counts[HealthStatus.UNHEALTHY] > 0:
+            overall_status = HealthStatus.UNHEALTHY
+        elif status_counts[HealthStatus.DEGRADED] > 0:
+            overall_status = HealthStatus.DEGRADED
+        elif status_counts[HealthStatus.HEALTHY] == len(self._components):
+            overall_status = HealthStatus.HEALTHY
+        else:
+            overall_status = HealthStatus.UNKNOWN
+        
+        return {
+            'overall_status': overall_status,
+            'components': components_data,
+            'summary': {
+                'total': len(self._components),
+                'healthy': status_counts[HealthStatus.HEALTHY],
+                'degraded': status_counts[HealthStatus.DEGRADED],
+                'unhealthy': status_counts[HealthStatus.UNHEALTHY],
+                'unknown': status_counts[HealthStatus.UNKNOWN]
+            }
+        }
+
+
+# Global health checker instance
+_global_health_checker = HealthChecker()
+
+
+def get_health_checker() -> HealthChecker:
+    """Get global health checker instance."""
+    return _global_health_checker
+
+
+def health_check_decorator(component_name: str):
+    """
+    Decorator to register function as health check.
+    
+    Args:
+        component_name: Name of component to monitor
+    """
+    def decorator(func: Callable) -> Callable:
+        _global_health_checker.register_component(component_name, func)
+        return func
+    
+    return decorator

unrealon_core/monitoring/metrics.py

@@ -0,0 +1,352 @@
+"""
+Metrics System
+
+Lightweight metrics collection and aggregation.
+Following critical requirements - max 500 lines, functions < 20 lines.
+
+Phase 2: Core Systems - Monitoring
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Dict, Any, List, Optional, Union
+from enum import Enum
+from collections import defaultdict, deque
+
+from pydantic import BaseModel, Field, ConfigDict
+
+from ..utils.time import utc_now
+
+
+logger = logging.getLogger(__name__)
+
+
+class MetricType(str, Enum):
+    """Metric types."""
+    COUNTER = "counter"
+    GAUGE = "gauge"
+    HISTOGRAM = "histogram"
+    TIMER = "timer"
+
+
+class MetricValue(BaseModel):
+    """Single metric value with timestamp."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    value: float = Field(description="Metric value")
+    timestamp: datetime = Field(description="When value was recorded")
+    labels: Dict[str, str] = Field(default_factory=dict, description="Metric labels")
+
+
+class Metric(BaseModel):
+    """Metric definition and current state."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    name: str = Field(description="Metric name")
+    metric_type: MetricType = Field(description="Type of metric")
+    description: str = Field(description="Metric description")
+    current_value: float = Field(default=0.0, description="Current metric value")
+    total_samples: int = Field(default=0, description="Total samples recorded")
+    last_updated: datetime = Field(description="Last update timestamp")
+    labels: Dict[str, str] = Field(default_factory=dict, description="Default labels")
+    
+    # Histogram-specific fields
+    buckets: Optional[List[float]] = Field(default=None, description="Histogram buckets")
+    bucket_counts: Optional[Dict[str, int]] = Field(default=None, description="Bucket counts")
+    
+    def update_value(self, value: float, labels: Optional[Dict[str, str]] = None) -> None:
+        """Update metric value."""
+        self.current_value = value
+        self.total_samples += 1
+        self.last_updated = utc_now()
+        
+        if labels:
+            self.labels.update(labels)
+    
+    def increment(self, amount: float = 1.0) -> None:
+        """Increment counter metric."""
+        if self.metric_type == MetricType.COUNTER:
+            self.current_value += amount
+            self.total_samples += 1
+            self.last_updated = utc_now()
+    
+    def observe_histogram(self, value: float) -> None:
+        """Observe value for histogram metric."""
+        if self.metric_type != MetricType.HISTOGRAM or not self.buckets:
+            return
+        
+        if not self.bucket_counts:
+            self.bucket_counts = {str(bucket): 0 for bucket in self.buckets}
+        
+        # Find appropriate bucket
+        for bucket in self.buckets:
+            if value <= bucket:
+                self.bucket_counts[str(bucket)] += 1
+        
+        self.total_samples += 1
+        self.last_updated = utc_now()
+
+
+class MetricsCollector:
+    """
+    Lightweight metrics collector.
+    
+    Collects and aggregates metrics for monitoring and alerting.
+    Designed for simplicity and low overhead.
+    """
+    
+    def __init__(self, max_history: int = 1000):
+        """Initialize metrics collector."""
+        self.max_history = max_history
+        self._metrics: Dict[str, Metric] = {}
+        self._history: Dict[str, deque] = defaultdict(lambda: deque(maxlen=max_history))
+        self._lock = asyncio.Lock()
+        self.logger = logging.getLogger("metrics_collector")
+    
+    async def register_metric(
+        self,
+        name: str,
+        metric_type: MetricType,
+        description: str,
+        labels: Optional[Dict[str, str]] = None,
+        buckets: Optional[List[float]] = None
+    ) -> None:
+        """
+        Register new metric.
+        
+        Args:
+            name: Metric name
+            metric_type: Type of metric
+            description: Metric description
+            labels: Default labels
+            buckets: Histogram buckets (for histogram metrics)
+        """
+        async with self._lock:
+            if name in self._metrics:
+                self.logger.warning(f"Metric {name} already registered")
+                return
+            
+            metric = Metric(
+                name=name,
+                metric_type=metric_type,
+                description=description,
+                last_updated=utc_now(),
+                labels=labels or {},
+                buckets=buckets,
+                bucket_counts={str(b): 0 for b in buckets} if buckets else None
+            )
+            
+            self._metrics[name] = metric
+            self.logger.debug(f"Registered metric: {name} ({metric_type})")
+    
+    async def record_counter(
+        self, 
+        name: str, 
+        value: float = 1.0,
+        labels: Optional[Dict[str, str]] = None
+    ) -> None:
+        """Record counter metric."""
+        await self._record_metric(name, value, MetricType.COUNTER, labels)
+    
+    async def record_gauge(
+        self, 
+        name: str, 
+        value: float,
+        labels: Optional[Dict[str, str]] = None
+    ) -> None:
+        """Record gauge metric."""
+        await self._record_metric(name, value, MetricType.GAUGE, labels)
+    
+    async def record_histogram(
+        self, 
+        name: str, 
+        value: float,
+        labels: Optional[Dict[str, str]] = None
+    ) -> None:
+        """Record histogram metric."""
+        await self._record_metric(name, value, MetricType.HISTOGRAM, labels)
+    
+    async def record_timer(
+        self, 
+        name: str, 
+        duration_seconds: float,
+        labels: Optional[Dict[str, str]] = None
+    ) -> None:
+        """Record timer metric."""
+        await self._record_metric(name, duration_seconds, MetricType.TIMER, labels)
+    
+    async def _record_metric(
+        self,
+        name: str,
+        value: float,
+        expected_type: MetricType,
+        labels: Optional[Dict[str, str]] = None
+    ) -> None:
+        """Internal method to record metric value."""
+        async with self._lock:
+            if name not in self._metrics:
+                # Auto-register metric
+                await self.register_metric(name, expected_type, f"Auto-registered {expected_type} metric")
+            
+            metric = self._metrics[name]
+            
+            if metric.metric_type != expected_type:
+                self.logger.error(f"Metric {name} type mismatch: expected {expected_type}, got {metric.metric_type}")
+                return
+            
+            # Update metric based on type
+            if metric.metric_type == MetricType.COUNTER:
+                metric.increment(value)
+            elif metric.metric_type == MetricType.HISTOGRAM:
+                metric.observe_histogram(value)
+            else:
+                metric.update_value(value, labels)
+            
+            # Store in history
+            metric_value = MetricValue(
+                value=value,
+                timestamp=utc_now(),
+                labels=labels or {}
+            )
+            
+            self._history[name].append(metric_value)
+    
+    async def get_metric(self, name: str) -> Optional[Metric]:
+        """Get metric by name."""
+        async with self._lock:
+            return self._metrics.get(name)
+    
+    async def get_all_metrics(self) -> Dict[str, Metric]:
+        """Get all registered metrics."""
+        async with self._lock:
+            return self._metrics.copy()
+    
+    async def get_metric_history(
+        self, 
+        name: str, 
+        limit: Optional[int] = None
+    ) -> List[MetricValue]:
+        """Get metric history."""
+        async with self._lock:
+            history = list(self._history.get(name, []))
+            
+            if limit:
+                history = history[-limit:]
+            
+            return history
+    
+    async def get_metrics_summary(self) -> Dict[str, Any]:
+        """Get summary of all metrics."""
+        async with self._lock:
+            summary = {
+                'total_metrics': len(self._metrics),
+                'metrics_by_type': defaultdict(int),
+                'metrics': {}
+            }
+            
+            for name, metric in self._metrics.items():
+                summary['metrics_by_type'][metric.metric_type] += 1
+                
+                summary['metrics'][name] = {
+                    'type': metric.metric_type,
+                    'current_value': metric.current_value,
+                    'total_samples': metric.total_samples,
+                    'last_updated': metric.last_updated.isoformat(),
+                    'description': metric.description
+                }
+                
+                # Add histogram-specific info
+                if metric.metric_type == MetricType.HISTOGRAM and metric.bucket_counts:
+                    summary['metrics'][name]['buckets'] = metric.bucket_counts
+            
+            return summary
+    
+    async def reset_metric(self, name: str) -> bool:
+        """Reset metric to initial state."""
+        async with self._lock:
+            if name not in self._metrics:
+                return False
+            
+            metric = self._metrics[name]
+            metric.current_value = 0.0
+            metric.total_samples = 0
+            metric.last_updated = utc_now()
+            
+            if metric.bucket_counts:
+                metric.bucket_counts = {bucket: 0 for bucket in metric.bucket_counts}
+            
+            # Clear history
+            self._history[name].clear()
+            
+            self.logger.info(f"Reset metric: {name}")
+            return True
+    
+    async def clear_all_metrics(self) -> None:
+        """Clear all metrics and history."""
+        async with self._lock:
+            self._metrics.clear()
+            self._history.clear()
+            self.logger.info("Cleared all metrics")
+
+
+# Global metrics collector
+_global_metrics = MetricsCollector()
+
+
+def get_metrics_collector() -> MetricsCollector:
+    """Get global metrics collector."""
+    return _global_metrics
+
+
+def reset_global_metrics() -> None:
+    """Reset global metrics collector (for testing)."""
+    global _global_metrics
+    _global_metrics = MetricsCollector()
+
+
+# Convenience functions
+async def counter(name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None) -> None:
+    """Record counter metric."""
+    await _global_metrics.record_counter(name, value, labels)
+
+
+async def gauge(name: str, value: float, labels: Optional[Dict[str, str]] = None) -> None:
+    """Record gauge metric."""
+    await _global_metrics.record_gauge(name, value, labels)
+
+
+async def histogram(name: str, value: float, labels: Optional[Dict[str, str]] = None) -> None:
+    """Record histogram metric."""
+    await _global_metrics.record_histogram(name, value, labels)
+
+
+class MetricTimer:
+    """Context manager for timing operations."""
+    
+    def __init__(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
+        self.metric_name = metric_name
+        self.labels = labels
+        self.start_time: Optional[datetime] = None
+    
+    async def __aenter__(self):
+        self.start_time = utc_now()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.start_time:
+            duration = (utc_now() - self.start_time).total_seconds()
+            await _global_metrics.record_timer(self.metric_name, duration, self.labels)
+
+
+def timer(metric_name: str, labels: Optional[Dict[str, str]] = None) -> MetricTimer:
+    """Create metric timer context manager."""
+    return MetricTimer(metric_name, labels)