agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/framework/resilience/circuit_breaker.py

@@ -0,0 +1,309 @@
+"""
+Circuit breaker implementation for fault tolerance.
+
+Prevents cascading failures by stopping requests to failing backends
+and allowing them to recover.
+"""
+
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Callable, Deque, Optional, TypeVar
+
+from .types import (
+    CircuitBreakerConfig,
+    CircuitOpenError,
+    CircuitState,
+    EventCallback,
+    ResilienceEvent,
+    ResilienceEventType,
+)
+
+T = TypeVar("T")
+
+
+@dataclass
+class CircuitStats:
+    """Statistics for circuit breaker."""
+
+    total_requests: int = 0
+    successful_requests: int = 0
+    failed_requests: int = 0
+    rejected_requests: int = 0
+    last_failure_time: Optional[float] = None
+    last_success_time: Optional[float] = None
+    state_changes: int = 0
+
+
+class CircuitBreaker:
+    """
+    Thread-safe circuit breaker implementation.
+
+    States:
+    - CLOSED: Normal operation, requests pass through
+    - OPEN: Failures exceeded threshold, requests rejected
+    - HALF_OPEN: Testing recovery, limited requests allowed
+
+    Example:
+        config = CircuitBreakerConfig(failure_threshold=5, timeout_seconds=30)
+        cb = CircuitBreaker("my_backend", config)
+
+        try:
+            result = cb.execute(lambda: backend.call())
+        except CircuitOpenError:
+            # Handle circuit open
+            pass
+    """
+
+    def __init__(
+        self,
+        name: str,
+        config: Optional[CircuitBreakerConfig] = None,
+        on_state_change: Optional[Callable[[CircuitState, CircuitState], None]] = None,
+        event_callback: Optional[EventCallback] = None,
+    ):
+        """
+        Initialize circuit breaker.
+
+        Args:
+            name: Name for this circuit breaker (typically backend name)
+            config: Configuration options
+            on_state_change: Callback for state transitions
+            event_callback: Callback for resilience events
+        """
+        self.name = name
+        self.config = config or CircuitBreakerConfig()
+        self.on_state_change = on_state_change
+        self.event_callback = event_callback
+
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._success_count = 0
+        self._half_open_requests = 0
+        self._last_failure_time: Optional[float] = None
+        self._lock = threading.RLock()
+
+        # Sliding window for failure rate calculation
+        self._results: Deque[bool] = deque(maxlen=self.config.window_size)
+
+        # Statistics
+        self.stats = CircuitStats()
+
+    @property
+    def state(self) -> CircuitState:
+        """Get current circuit state (checks for automatic transitions)."""
+        with self._lock:
+            self._check_state_transition()
+            return self._state
+
+    @property
+    def is_closed(self) -> bool:
+        """Check if circuit is closed (normal operation)."""
+        return self.state == CircuitState.CLOSED
+
+    @property
+    def is_open(self) -> bool:
+        """Check if circuit is open (rejecting requests)."""
+        return self.state == CircuitState.OPEN
+
+    @property
+    def is_half_open(self) -> bool:
+        """Check if circuit is half-open (testing recovery)."""
+        return self.state == CircuitState.HALF_OPEN
+
+    def execute(self, func: Callable[[], T]) -> T:
+        """
+        Execute function through circuit breaker.
+
+        Args:
+            func: Function to execute
+
+        Returns:
+            Function result
+
+        Raises:
+            CircuitOpenError: If circuit is open
+        """
+        with self._lock:
+            self._check_state_transition()
+
+            if self._state == CircuitState.OPEN:
+                time_until_retry = self._time_until_retry()
+                self.stats.rejected_requests += 1
+                raise CircuitOpenError(self.name, time_until_retry)
+
+            if self._state == CircuitState.HALF_OPEN:
+                if self._half_open_requests >= self.config.half_open_max_requests:
+                    raise CircuitOpenError(self.name, 0)
+                self._half_open_requests += 1
+
+            self.stats.total_requests += 1
+
+        # Execute outside lock to avoid blocking other threads
+        try:
+            result = func()
+            self._record_success()
+            return result
+        except Exception as e:
+            if type(e) not in self.config.excluded_exceptions:
+                self._record_failure()
+            raise
+
+    def _record_success(self) -> None:
+        """Record a successful request."""
+        with self._lock:
+            self._results.append(True)
+            self._success_count += 1
+            self.stats.successful_requests += 1
+            self.stats.last_success_time = time.monotonic()
+
+            if self._state == CircuitState.HALF_OPEN:
+                if self._success_count >= self.config.success_threshold:
+                    self._transition_to(CircuitState.CLOSED)
+
+    def _record_failure(self) -> None:
+        """Record a failed request."""
+        with self._lock:
+            self._results.append(False)
+            self._failure_count += 1
+            self._last_failure_time = time.monotonic()
+            self.stats.failed_requests += 1
+            self.stats.last_failure_time = time.monotonic()
+
+            if self._state == CircuitState.HALF_OPEN:
+                # Any failure in half-open trips back to open
+                self._transition_to(CircuitState.OPEN)
+            elif self._state == CircuitState.CLOSED:
+                if self._should_trip():
+                    self._transition_to(CircuitState.OPEN)
+
+    def _should_trip(self) -> bool:
+        """Check if circuit should trip to OPEN."""
+        # Check absolute failure count
+        if self._failure_count >= self.config.failure_threshold:
+            return True
+
+        # Check failure rate if window is full
+        if len(self._results) >= self.config.window_size:
+            failures = sum(1 for r in self._results if not r)
+            failure_rate = failures / len(self._results)
+            if failure_rate >= self.config.failure_rate_threshold:
+                return True
+
+        return False
+
+    def _check_state_transition(self) -> None:
+        """Check for automatic state transitions (OPEN -> HALF_OPEN)."""
+        if self._state == CircuitState.OPEN:
+            if self._last_failure_time is not None:
+                elapsed = time.monotonic() - self._last_failure_time
+                if elapsed >= self.config.timeout_seconds:
+                    self._transition_to(CircuitState.HALF_OPEN)
+
+    def _transition_to(self, new_state: CircuitState) -> None:
+        """Transition to a new state."""
+        old_state = self._state
+        self._state = new_state
+        self.stats.state_changes += 1
+
+        # Reset counters based on new state
+        if new_state == CircuitState.CLOSED:
+            self._failure_count = 0
+            self._success_count = 0
+            self._results.clear()
+        elif new_state == CircuitState.HALF_OPEN:
+            self._success_count = 0
+            self._half_open_requests = 0
+        elif new_state == CircuitState.OPEN:
+            self._last_failure_time = time.monotonic()
+
+        # Callbacks
+        if self.on_state_change:
+            try:
+                self.on_state_change(old_state, new_state)
+            except Exception:
+                pass
+
+        self._emit_event(old_state, new_state)
+
+    def _time_until_retry(self) -> float:
+        """Get time until circuit will transition to half-open."""
+        if self._last_failure_time is None:
+            return 0
+        elapsed = time.monotonic() - self._last_failure_time
+        return max(0, self.config.timeout_seconds - elapsed)
+
+    def _emit_event(self, old_state: CircuitState, new_state: CircuitState) -> None:
+        """Emit state change event."""
+        if not self.event_callback:
+            return
+
+        event_type_map = {
+            CircuitState.OPEN: ResilienceEventType.CIRCUIT_OPENED,
+            CircuitState.CLOSED: ResilienceEventType.CIRCUIT_CLOSED,
+            CircuitState.HALF_OPEN: ResilienceEventType.CIRCUIT_HALF_OPEN,
+        }
+
+        event = ResilienceEvent(
+            event_type=event_type_map[new_state],
+            backend_name=self.name,
+            metadata={
+                "old_state": old_state.value,
+                "new_state": new_state.value,
+                "failure_count": self._failure_count,
+            },
+        )
+
+        try:
+            self.event_callback(event)
+        except Exception:
+            pass
+
+    def reset(self) -> None:
+        """Reset circuit breaker to closed state."""
+        with self._lock:
+            old_state = self._state
+            self._state = CircuitState.CLOSED
+            self._failure_count = 0
+            self._success_count = 0
+            self._half_open_requests = 0
+            self._results.clear()
+            self._last_failure_time = None
+            self.stats.state_changes += 1
+
+            if old_state != CircuitState.CLOSED:
+                self._emit_event(old_state, CircuitState.CLOSED)
+                if self.on_state_change:
+                    try:
+                        self.on_state_change(old_state, CircuitState.CLOSED)
+                    except Exception:
+                        pass
+
+    def force_open(self) -> None:
+        """Force circuit to open state (for testing/maintenance)."""
+        with self._lock:
+            if self._state != CircuitState.OPEN:
+                self._transition_to(CircuitState.OPEN)
+
+    def get_failure_rate(self) -> float:
+        """Get current failure rate from sliding window."""
+        with self._lock:
+            if not self._results:
+                return 0.0
+            failures = sum(1 for r in self._results if not r)
+            return failures / len(self._results)
+
+    def get_stats(self) -> dict:
+        """Get circuit breaker statistics as dictionary."""
+        with self._lock:
+            return {
+                "state": self._state.value,
+                "total_requests": self.stats.total_requests,
+                "successful_requests": self.stats.successful_requests,
+                "failed_requests": self.stats.failed_requests,
+                "rejected_requests": self.stats.rejected_requests,
+                "state_changes": self.stats.state_changes,
+                "failure_rate": self.get_failure_rate(),
+                "failure_count": self._failure_count,
+            }

fi/evals/framework/resilience/degradation.py

@@ -0,0 +1,355 @@
+"""
+Graceful degradation with fallback chains.
+
+Provides fallback mechanisms when primary operations fail.
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import Callable, Generic, List, Optional, TypeVar
+
+from .types import (
+    DegradationConfig,
+    EventCallback,
+    ResilienceEvent,
+    ResilienceEventType,
+)
+
+T = TypeVar("T")
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FallbackStats:
+    """Statistics for fallback handler."""
+
+    total_calls: int = 0
+    primary_success: int = 0
+    fallback_used: int = 0
+    all_failed: int = 0
+
+
+@dataclass
+class FallbackOption(Generic[T]):
+    """
+    A fallback option with name and callable.
+
+    Attributes:
+        name: Identifier for this fallback
+        func: Callable that returns the fallback value
+        condition: Optional condition to check before using this fallback
+    """
+
+    name: str
+    func: Callable[[], T]
+    condition: Optional[Callable[[Exception], bool]] = None
+
+
+class FallbackChain(Generic[T]):
+    """
+    Fallback chain for graceful degradation.
+
+    Tries a series of fallbacks in order when the primary operation fails.
+
+    Example:
+        chain = FallbackChain("api_call")
+        chain.add_fallback("cache", lambda: get_from_cache())
+        chain.add_fallback("default", lambda: default_value)
+
+        result = chain.execute(lambda: api.call())
+    """
+
+    def __init__(
+        self,
+        name: str,
+        config: Optional[DegradationConfig] = None,
+        event_callback: Optional[EventCallback] = None,
+    ):
+        """
+        Initialize fallback chain.
+
+        Args:
+            name: Name for this fallback chain
+            config: Degradation configuration
+            event_callback: Callback for degradation events
+        """
+        self.name = name
+        self.config = config or DegradationConfig()
+        self.event_callback = event_callback
+
+        self._fallbacks: List[FallbackOption[T]] = []
+        self.stats = FallbackStats()
+
+    def add_fallback(
+        self,
+        name: str,
+        func: Callable[[], T],
+        condition: Optional[Callable[[Exception], bool]] = None,
+    ) -> "FallbackChain[T]":
+        """
+        Add a fallback option to the chain.
+
+        Args:
+            name: Name for this fallback
+            func: Callable that returns the fallback value
+            condition: Optional condition (receives exception, returns bool)
+
+        Returns:
+            Self for chaining
+        """
+        self._fallbacks.append(FallbackOption(name=name, func=func, condition=condition))
+        return self
+
+    def execute(
+        self,
+        primary: Callable[[], T],
+        fallback_exceptions: Optional[set[type]] = None,
+    ) -> T:
+        """
+        Execute primary with fallbacks on failure.
+
+        Args:
+            primary: Primary callable to execute
+            fallback_exceptions: Exception types that trigger fallback
+                                (defaults to config.fallback_exceptions)
+
+        Returns:
+            Result from primary or a fallback
+
+        Raises:
+            Exception: If primary and all fallbacks fail
+        """
+        exceptions = fallback_exceptions or self.config.fallback_on_exceptions
+        self.stats.total_calls += 1
+
+        # Try primary
+        try:
+            result = primary()
+            self.stats.primary_success += 1
+            return result
+        except Exception as e:
+            if not self._should_fallback(e, exceptions):
+                raise
+
+            primary_error = e
+
+        # Try fallbacks in order
+        for fallback in self._fallbacks:
+            # Check condition if specified
+            if fallback.condition and not fallback.condition(primary_error):
+                continue
+
+            try:
+                result = fallback.func()
+                self.stats.fallback_used += 1
+                self._emit_fallback_event(fallback.name, primary_error)
+                return result
+            except Exception as fallback_error:
+                logger.debug(
+                    f"Fallback '{fallback.name}' failed: {fallback_error}"
+                )
+                continue
+
+        # All failed
+        self.stats.all_failed += 1
+        self._emit_all_failed_event(primary_error)
+        raise primary_error
+
+    def _should_fallback(
+        self, exception: Exception, exceptions: set[type]
+    ) -> bool:
+        """Check if exception should trigger fallback."""
+        if not exceptions:
+            return True  # Fallback on all exceptions
+
+        for exc_type in exceptions:
+            if isinstance(exception, exc_type):
+                return True
+        return False
+
+    def _emit_fallback_event(self, fallback_name: str, error: Exception) -> None:
+        """Emit fallback used event."""
+        if not self.event_callback:
+            return
+
+        event = ResilienceEvent(
+            event_type=ResilienceEventType.FALLBACK_USED,
+            backend_name=self.name,
+            metadata={
+                "fallback_name": fallback_name,
+                "primary_error": str(error),
+                "error_type": type(error).__name__,
+            },
+        )
+
+        try:
+            self.event_callback(event)
+        except Exception:
+            pass
+
+    def _emit_all_failed_event(self, error: Exception) -> None:
+        """Emit all fallbacks failed event."""
+        if not self.event_callback:
+            return
+
+        event = ResilienceEvent(
+            event_type=ResilienceEventType.DEGRADATION_ACTIVE,
+            backend_name=self.name,
+            metadata={
+                "fallback_count": len(self._fallbacks),
+                "all_failed": True,
+                "last_error": str(error),
+            },
+        )
+
+        try:
+            self.event_callback(event)
+        except Exception:
+            pass
+
+    def get_stats(self) -> dict:
+        """Get fallback chain statistics as dictionary."""
+        return {
+            "total_calls": self.stats.total_calls,
+            "primary_success": self.stats.primary_success,
+            "fallback_used": self.stats.fallback_used,
+            "all_failed": self.stats.all_failed,
+            "fallback_rate": (
+                self.stats.fallback_used / self.stats.total_calls
+                if self.stats.total_calls > 0
+                else 0.0
+            ),
+        }
+
+
+class DegradationHandler:
+    """
+    Handler for graceful degradation with primary/secondary pattern.
+
+    Manages a primary operation with a single fallback operation.
+
+    Example:
+        handler = DegradationHandler(
+            "api",
+            primary=lambda: api.call(),
+            fallback=lambda: cache.get(),
+        )
+
+        result = handler.execute()
+    """
+
+    def __init__(
+        self,
+        name: str,
+        primary: Callable[[], T],
+        fallback: Callable[[], T],
+        config: Optional[DegradationConfig] = None,
+        event_callback: Optional[EventCallback] = None,
+    ):
+        """
+        Initialize degradation handler.
+
+        Args:
+            name: Name for this handler
+            primary: Primary callable
+            fallback: Fallback callable
+            config: Degradation configuration
+            event_callback: Callback for events
+        """
+        self.name = name
+        self._primary = primary
+        self._fallback = fallback
+        self.config = config or DegradationConfig()
+        self.event_callback = event_callback
+
+        self._chain: FallbackChain = FallbackChain(
+            name, config, event_callback
+        )
+        self._chain.add_fallback("fallback", fallback)
+
+    def execute(self) -> T:
+        """
+        Execute primary with fallback on failure.
+
+        Returns:
+            Result from primary or fallback
+
+        Raises:
+            Exception: If both primary and fallback fail
+        """
+        return self._chain.execute(self._primary)
+
+    @property
+    def stats(self) -> FallbackStats:
+        """Get statistics."""
+        return self._chain.stats
+
+    def get_stats(self) -> dict:
+        """Get statistics as dictionary."""
+        return self._chain.get_stats()
+
+
+def with_fallback(
+    fallback_value: T,
+    exceptions: Optional[set[type]] = None,
+    name: Optional[str] = None,
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """
+    Decorator for adding a static fallback value.
+
+    Example:
+        @with_fallback(default_config)
+        def load_config():
+            return api.get_config()
+    """
+
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        handler_name = name or func.__name__
+        chain: FallbackChain = FallbackChain(handler_name)
+        chain.add_fallback("default", lambda: fallback_value)
+
+        def wrapper(*args, **kwargs) -> T:
+            return chain.execute(
+                lambda: func(*args, **kwargs),
+                fallback_exceptions=exceptions,
+            )
+
+        wrapper.__name__ = func.__name__
+        wrapper.__doc__ = func.__doc__
+        wrapper._fallback_chain = chain  # type: ignore
+        return wrapper
+
+    return decorator
+
+
+def with_fallback_func(
+    fallback_func: Callable[[], T],
+    exceptions: Optional[set[type]] = None,
+    name: Optional[str] = None,
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """
+    Decorator for adding a fallback function.
+
+    Example:
+        @with_fallback_func(lambda: load_from_cache())
+        def load_config():
+            return api.get_config()
+    """
+
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        handler_name = name or func.__name__
+        chain: FallbackChain = FallbackChain(handler_name)
+        chain.add_fallback("fallback", fallback_func)
+
+        def wrapper(*args, **kwargs) -> T:
+            return chain.execute(
+                lambda: func(*args, **kwargs),
+                fallback_exceptions=exceptions,
+            )
+
+        wrapper.__name__ = func.__name__
+        wrapper.__doc__ = func.__doc__
+        wrapper._fallback_chain = chain  # type: ignore
+        return wrapper
+
+    return decorator