unrealon 1.1.5__py3-none-any.whl → 2.0.4__py3-none-any.whl

unrealon_core/error_handling/circuit_breaker.py

@@ -0,0 +1,292 @@
+"""
+Circuit Breaker Pattern
+
+Prevents cascading failures by temporarily disabling failing services.
+Following critical requirements - max 500 lines, functions < 20 lines.
+
+Phase 2: Core Systems - Error Handling
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Callable, Optional, Dict
+from enum import Enum
+
+from pydantic import BaseModel, Field, ConfigDict
+
+from ..exceptions.base import UnrealOnError
+from ..utils.time import utc_now
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitBreakerState(str, Enum):
+    """Circuit breaker states."""
+    CLOSED = "closed"      # Normal operation
+    OPEN = "open"          # Failing, requests blocked
+    HALF_OPEN = "half_open"  # Testing if service recovered
+
+
+class CircuitBreakerError(UnrealOnError):
+    """Circuit breaker is open."""
+    pass
+
+
+class CircuitBreakerConfig(BaseModel):
+    """Configuration for circuit breaker."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    failure_threshold: int = Field(
+        default=5, 
+        ge=1, 
+        le=20, 
+        description="Failures needed to open circuit"
+    )
+    recovery_timeout: float = Field(
+        default=60.0, 
+        ge=1.0, 
+        le=300.0, 
+        description="Seconds before trying half-open"
+    )
+    success_threshold: int = Field(
+        default=3, 
+        ge=1, 
+        le=10, 
+        description="Successes needed to close circuit"
+    )
+    timeout: float = Field(
+        default=30.0, 
+        ge=1.0, 
+        le=120.0, 
+        description="Operation timeout in seconds"
+    )
+
+
+class CircuitBreakerStats(BaseModel):
+    """Circuit breaker statistics."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    total_requests: int = Field(default=0, description="Total requests processed")
+    successful_requests: int = Field(default=0, description="Successful requests")
+    failed_requests: int = Field(default=0, description="Failed requests")
+    rejected_requests: int = Field(default=0, description="Requests rejected by circuit")
+    last_failure_time: Optional[datetime] = Field(default=None, description="Last failure timestamp")
+    last_success_time: Optional[datetime] = Field(default=None, description="Last success timestamp")
+    
+    def get_failure_rate(self) -> float:
+        """Calculate failure rate percentage."""
+        if self.total_requests == 0:
+            return 0.0
+        return (self.failed_requests / self.total_requests) * 100.0
+
+
+class CircuitBreaker:
+    """
+    Circuit breaker implementation.
+    
+    Prevents cascading failures by monitoring service health
+    and temporarily blocking requests to failing services.
+    """
+    
+    def __init__(self, name: str, config: CircuitBreakerConfig):
+        """Initialize circuit breaker."""
+        self.name = name
+        self.config = config
+        self.state = CircuitBreakerState.CLOSED
+        self.stats = CircuitBreakerStats()
+        
+        # State tracking
+        self._consecutive_failures = 0
+        self._consecutive_successes = 0
+        self._last_failure_time: Optional[datetime] = None
+        self._lock = asyncio.Lock()
+        
+        self.logger = logging.getLogger(f"circuit_breaker.{name}")
+    
+    async def call(self, func: Callable[..., Any], *args, **kwargs) -> Any:
+        """
+        Execute function through circuit breaker.
+        
+        Args:
+            func: Function to execute
+            *args, **kwargs: Function arguments
+            
+        Returns:
+            Function result
+            
+        Raises:
+            CircuitBreakerError: If circuit is open
+        """
+        async with self._lock:
+            await self._update_state()
+            
+            if self.state == CircuitBreakerState.OPEN:
+                self.stats.rejected_requests += 1
+                self.logger.warning(f"Circuit breaker {self.name} is OPEN - rejecting request")
+                raise CircuitBreakerError(f"Circuit breaker {self.name} is open")
+            
+            self.stats.total_requests += 1
+        
+        # Execute function with timeout
+        try:
+            result = await asyncio.wait_for(
+                func(*args, **kwargs),
+                timeout=self.config.timeout
+            )
+            
+            await self._record_success()
+            return result
+            
+        except Exception as e:
+            await self._record_failure()
+            raise
+    
+    async def _update_state(self) -> None:
+        """Update circuit breaker state based on current conditions."""
+        now = utc_now()
+        
+        if self.state == CircuitBreakerState.OPEN:
+            # Check if we should transition to half-open
+            if (self._last_failure_time and 
+                (now - self._last_failure_time).total_seconds() >= self.config.recovery_timeout):
+                
+                self.state = CircuitBreakerState.HALF_OPEN
+                self._consecutive_successes = 0
+                self.logger.info(f"Circuit breaker {self.name} transitioning to HALF_OPEN")
+        
+        elif self.state == CircuitBreakerState.HALF_OPEN:
+            # Check if we should close the circuit
+            if self._consecutive_successes >= self.config.success_threshold:
+                self.state = CircuitBreakerState.CLOSED
+                self._consecutive_failures = 0
+                self.logger.info(f"Circuit breaker {self.name} transitioning to CLOSED")
+    
+    async def _record_success(self) -> None:
+        """Record successful operation."""
+        async with self._lock:
+            self.stats.successful_requests += 1
+            self.stats.last_success_time = utc_now()
+            
+            if self.state == CircuitBreakerState.HALF_OPEN:
+                self._consecutive_successes += 1
+                self.logger.debug(
+                    f"Circuit breaker {self.name} success "
+                    f"({self._consecutive_successes}/{self.config.success_threshold})"
+                )
+            
+            # Reset failure counter on success
+            self._consecutive_failures = 0
+    
+    async def _record_failure(self) -> None:
+        """Record failed operation."""
+        async with self._lock:
+            self.stats.failed_requests += 1
+            self.stats.last_failure_time = utc_now()
+            self._last_failure_time = self.stats.last_failure_time
+            
+            self._consecutive_failures += 1
+            self._consecutive_successes = 0
+            
+            self.logger.debug(
+                f"Circuit breaker {self.name} failure "
+                f"({self._consecutive_failures}/{self.config.failure_threshold})"
+            )
+            
+            # Check if we should open the circuit
+            if (self.state == CircuitBreakerState.CLOSED and 
+                self._consecutive_failures >= self.config.failure_threshold):
+                
+                self.state = CircuitBreakerState.OPEN
+                self.logger.warning(f"Circuit breaker {self.name} transitioning to OPEN")
+            
+            elif self.state == CircuitBreakerState.HALF_OPEN:
+                # Any failure in half-open state reopens the circuit
+                self.state = CircuitBreakerState.OPEN
+                self.logger.warning(f"Circuit breaker {self.name} reopening due to failure in HALF_OPEN")
+    
+    def get_status(self) -> Dict[str, Any]:
+        """Get circuit breaker status."""
+        return {
+            'name': self.name,
+            'state': self.state.value,
+            'consecutive_failures': self._consecutive_failures,
+            'consecutive_successes': self._consecutive_successes,
+            'config': self.config.model_dump(),
+            'stats': self.stats.model_dump()
+        }
+    
+    def reset(self) -> None:
+        """Reset circuit breaker to closed state."""
+        self.state = CircuitBreakerState.CLOSED
+        self._consecutive_failures = 0
+        self._consecutive_successes = 0
+        self._last_failure_time = None
+        self.logger.info(f"Circuit breaker {self.name} manually reset to CLOSED")
+
+
+# Global circuit breaker registry
+_circuit_breakers: Dict[str, CircuitBreaker] = {}
+
+
+def get_circuit_breaker(
+    name: str, 
+    config: Optional[CircuitBreakerConfig] = None
+) -> CircuitBreaker:
+    """Get or create circuit breaker by name."""
+    if name not in _circuit_breakers:
+        if config is None:
+            config = CircuitBreakerConfig()
+        _circuit_breakers[name] = CircuitBreaker(name, config)
+    
+    return _circuit_breakers[name]
+
+
+def circuit_breaker(
+    name: str, 
+    config: Optional[CircuitBreakerConfig] = None
+):
+    """Decorator to wrap function with circuit breaker."""
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        cb = get_circuit_breaker(name, config)
+        
+        async def async_wrapper(*args, **kwargs):
+            return await cb.call(func, *args, **kwargs)
+        
+        def sync_wrapper(*args, **kwargs):
+            # For sync functions, we need to handle differently
+            # This is a simplified version - in production you might want
+            # to use threading or a different approach
+            import asyncio
+            try:
+                loop = asyncio.get_event_loop()
+                return loop.run_until_complete(cb.call(func, *args, **kwargs))
+            except RuntimeError:
+                # No event loop running
+                return asyncio.run(cb.call(func, *args, **kwargs))
+        
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator
+
+
+def get_all_circuit_breakers() -> Dict[str, Dict[str, Any]]:
+    """Get status of all circuit breakers."""
+    return {name: cb.get_status() for name, cb in _circuit_breakers.items()}
+
+
+def reset_all_circuit_breakers() -> None:
+    """Reset all circuit breakers."""
+    for cb in _circuit_breakers.values():
+        cb.reset()

unrealon_core/error_handling/error_context.py

@@ -0,0 +1,324 @@
+"""
+Error Context System
+
+Rich error context for debugging and monitoring.
+Following critical requirements - max 500 lines, functions < 20 lines.
+
+Phase 2: Core Systems - Error Handling
+"""
+
+import traceback
+from datetime import datetime
+from typing import Any, Dict, Optional, List
+from enum import Enum
+
+from pydantic import BaseModel, Field, ConfigDict
+
+from ..utils.time import utc_now
+
+
+class ErrorSeverity(str, Enum):
+    """Error severity levels."""
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+class ErrorContext(BaseModel):
+    """Rich error context information."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    # Basic error info
+    error_id: str = Field(description="Unique error identifier")
+    error_type: str = Field(description="Error class name")
+    error_message: str = Field(description="Error message")
+    severity: ErrorSeverity = Field(description="Error severity level")
+    
+    # Timing
+    timestamp: datetime = Field(description="When error occurred")
+    duration_ms: Optional[float] = Field(default=None, description="Operation duration before error")
+    
+    # Context
+    operation: str = Field(description="Operation that failed")
+    component: str = Field(description="Component where error occurred")
+    user_id: Optional[str] = Field(default=None, description="Associated user ID")
+    request_id: Optional[str] = Field(default=None, description="Request correlation ID")
+    
+    # Technical details
+    stack_trace: Optional[str] = Field(default=None, description="Full stack trace")
+    function_name: str = Field(description="Function where error occurred")
+    file_name: str = Field(description="File where error occurred")
+    line_number: int = Field(description="Line number where error occurred")
+    
+    # Additional context
+    parameters: Dict[str, Any] = Field(default_factory=dict, description="Function parameters")
+    environment: Dict[str, str] = Field(default_factory=dict, description="Environment variables")
+    metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
+    
+    # Recovery info
+    is_retryable: bool = Field(default=True, description="Whether error is retryable")
+    retry_count: int = Field(default=0, description="Number of retries attempted")
+    recovery_suggestions: List[str] = Field(default_factory=list, description="Recovery suggestions")
+
+
+def create_error_context(
+    error: Exception,
+    operation: str,
+    component: str,
+    severity: ErrorSeverity = ErrorSeverity.MEDIUM,
+    **kwargs
+) -> ErrorContext:
+    """
+    Create error context from exception.
+    
+    Args:
+        error: Exception that occurred
+        operation: Operation that failed
+        component: Component where error occurred
+        severity: Error severity level
+        **kwargs: Additional context data
+        
+    Returns:
+        ErrorContext with rich error information
+    """
+    import uuid
+    import inspect
+    import os
+    
+    # Get stack trace info
+    tb = traceback.extract_tb(error.__traceback__)
+    if tb:
+        frame = tb[-1]  # Last frame (where error occurred)
+        file_name = os.path.basename(frame.filename)
+        function_name = frame.name
+        line_number = frame.lineno
+    else:
+        file_name = "unknown"
+        function_name = "unknown"
+        line_number = 0
+    
+    # Generate error ID
+    error_id = str(uuid.uuid4())[:8]
+    
+    # Extract parameters from current frame
+    parameters = {}
+    try:
+        frame = inspect.currentframe()
+        if frame and frame.f_back:
+            parameters = {
+                k: str(v) for k, v in frame.f_back.f_locals.items()
+                if not k.startswith('_') and not callable(v)
+            }
+    except Exception:
+        pass  # Ignore parameter extraction errors
+    
+    # Basic environment info
+    environment = {
+        'python_version': f"{os.sys.version_info.major}.{os.sys.version_info.minor}",
+        'platform': os.sys.platform,
+    }
+    
+    return ErrorContext(
+        error_id=error_id,
+        error_type=error.__class__.__name__,
+        error_message=str(error),
+        severity=severity,
+        timestamp=utc_now(),
+        operation=operation,
+        component=component,
+        stack_trace=traceback.format_exc(),
+        function_name=function_name,
+        file_name=file_name,
+        line_number=line_number,
+        parameters=parameters,
+        environment=environment,
+        **kwargs
+    )
+
+
+def format_error_context(context: ErrorContext, include_stack_trace: bool = False) -> str:
+    """
+    Format error context for logging.
+    
+    Args:
+        context: Error context to format
+        include_stack_trace: Whether to include full stack trace
+        
+    Returns:
+        Formatted error message
+    """
+    lines = [
+        f"🚨 ERROR [{context.error_id}] {context.severity.upper()}",
+        f"   Type: {context.error_type}",
+        f"   Message: {context.error_message}",
+        f"   Operation: {context.operation}",
+        f"   Component: {context.component}",
+        f"   Location: {context.file_name}:{context.line_number} in {context.function_name}()",
+        f"   Time: {context.timestamp.isoformat()}",
+    ]
+    
+    if context.duration_ms:
+        lines.append(f"   Duration: {context.duration_ms:.2f}ms")
+    
+    if context.request_id:
+        lines.append(f"   Request ID: {context.request_id}")
+    
+    if context.user_id:
+        lines.append(f"   User ID: {context.user_id}")
+    
+    if context.retry_count > 0:
+        lines.append(f"   Retries: {context.retry_count}")
+    
+    if context.parameters:
+        lines.append("   Parameters:")
+        for key, value in context.parameters.items():
+            lines.append(f"     {key}: {value}")
+    
+    if context.recovery_suggestions:
+        lines.append("   Recovery suggestions:")
+        for suggestion in context.recovery_suggestions:
+            lines.append(f"     • {suggestion}")
+    
+    if include_stack_trace and context.stack_trace:
+        lines.append("   Stack trace:")
+        for line in context.stack_trace.split('\n'):
+            if line.strip():
+                lines.append(f"     {line}")
+    
+    return '\n'.join(lines)
+
+
+def determine_severity(error: Exception) -> ErrorSeverity:
+    """
+    Determine error severity based on exception type.
+    
+    Args:
+        error: Exception to analyze
+        
+    Returns:
+        Appropriate severity level
+    """
+    error_name = error.__class__.__name__
+    
+    # Critical errors
+    critical_errors = [
+        'SystemError', 'MemoryError', 'KeyboardInterrupt',
+        'SystemExit', 'GeneratorExit'
+    ]
+    
+    # High severity errors
+    high_errors = [
+        'ConnectionError', 'DatabaseError', 'AuthenticationError',
+        'PermissionError', 'SecurityError'
+    ]
+    
+    # Medium severity errors
+    medium_errors = [
+        'ValidationError', 'ValueError', 'TypeError',
+        'AttributeError', 'KeyError', 'IndexError'
+    ]
+    
+    if error_name in critical_errors:
+        return ErrorSeverity.CRITICAL
+    elif error_name in high_errors:
+        return ErrorSeverity.HIGH
+    elif error_name in medium_errors:
+        return ErrorSeverity.MEDIUM
+    else:
+        return ErrorSeverity.LOW
+
+
+def is_retryable_error(error: Exception) -> bool:
+    """
+    Determine if error is retryable.
+    
+    Args:
+        error: Exception to analyze
+        
+    Returns:
+        True if error should be retried
+    """
+    error_name = error.__class__.__name__
+    
+    # Non-retryable errors
+    non_retryable = [
+        'ValidationError', 'ValueError', 'TypeError',
+        'AttributeError', 'KeyError', 'IndexError',
+        'AuthenticationError', 'PermissionError',
+        'NotImplementedError', 'AssertionError'
+    ]
+    
+    # Retryable errors
+    retryable = [
+        'ConnectionError', 'TimeoutError', 'HTTPError',
+        'NetworkError', 'ServiceUnavailableError',
+        'TemporaryError', 'RateLimitError'
+    ]
+    
+    if error_name in non_retryable:
+        return False
+    elif error_name in retryable:
+        return True
+    else:
+        # Default to retryable for unknown errors
+        return True
+
+
+def get_recovery_suggestions(error: Exception, operation: str) -> List[str]:
+    """
+    Get recovery suggestions for error.
+    
+    Args:
+        error: Exception that occurred
+        operation: Operation that failed
+        
+    Returns:
+        List of recovery suggestions
+    """
+    error_name = error.__class__.__name__
+    suggestions = []
+    
+    if error_name == 'ConnectionError':
+        suggestions.extend([
+            "Check network connectivity",
+            "Verify service endpoint is accessible",
+            "Check firewall settings"
+        ])
+    
+    elif error_name == 'TimeoutError':
+        suggestions.extend([
+            "Increase timeout value",
+            "Check service performance",
+            "Retry with exponential backoff"
+        ])
+    
+    elif error_name == 'ValidationError':
+        suggestions.extend([
+            "Check input data format",
+            "Verify required fields are present",
+            "Review data validation rules"
+        ])
+    
+    elif error_name == 'AuthenticationError':
+        suggestions.extend([
+            "Check authentication credentials",
+            "Verify token is not expired",
+            "Refresh authentication token"
+        ])
+    
+    elif error_name == 'PermissionError':
+        suggestions.extend([
+            "Check user permissions",
+            "Verify access rights for operation",
+            "Contact administrator for access"
+        ])
+    
+    else:
+        suggestions.append(f"Review {operation} implementation")
+    
+    return suggestions