truthound 1.0.8__py3-none-any.whl

truthound/checkpoint/circuitbreaker/core.py

@@ -0,0 +1,252 @@
+"""Core types and configuration for Circuit Breaker pattern.
+
+This module defines:
+- Circuit states (CLOSED, OPEN, HALF_OPEN)
+- Configuration dataclasses
+- Exception types
+- Metrics and event types
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Callable
+
+
+class CircuitState(str, Enum):
+    """Circuit breaker states.
+
+    State transitions:
+        CLOSED -> OPEN: When failure threshold is exceeded
+        OPEN -> HALF_OPEN: After recovery timeout expires
+        HALF_OPEN -> CLOSED: When test calls succeed
+        HALF_OPEN -> OPEN: When test calls fail
+    """
+
+    CLOSED = "closed"
+    """Normal operation - requests pass through."""
+
+    OPEN = "open"
+    """Circuit tripped - requests fail immediately."""
+
+    HALF_OPEN = "half_open"
+    """Testing recovery - limited requests allowed."""
+
+
+class FailureDetectionStrategy(str, Enum):
+    """Strategy for detecting failures."""
+
+    CONSECUTIVE = "consecutive"
+    """Trip after N consecutive failures."""
+
+    PERCENTAGE = "percentage"
+    """Trip when failure percentage exceeds threshold."""
+
+    TIME_WINDOW = "time_window"
+    """Trip based on failures within a time window."""
+
+    COMPOSITE = "composite"
+    """Combine multiple detection strategies."""
+
+
+@dataclass
+class CircuitBreakerConfig:
+    """Configuration for circuit breaker behavior.
+
+    Attributes:
+        failure_threshold: Number of failures before opening circuit
+        recovery_timeout: Seconds to wait before attempting recovery
+        half_open_max_calls: Max calls allowed in half-open state
+        success_threshold: Successes needed to close from half-open
+        detection_strategy: Strategy for detecting failures
+        failure_rate_threshold: Failure rate to trip (for percentage strategy)
+        time_window_seconds: Window size for time-based detection
+        min_calls_in_window: Minimum calls before percentage applies
+        excluded_exceptions: Exception types that don't count as failures
+        included_exceptions: Only these exceptions count as failures (if set)
+        fallback: Optional fallback function when circuit is open
+        on_state_change: Callback for state changes
+        on_failure: Callback for failures
+        on_success: Callback for successes
+    """
+
+    # Thresholds
+    failure_threshold: int = 5
+    recovery_timeout: float = 30.0
+    half_open_max_calls: int = 3
+    success_threshold: int = 2
+
+    # Detection strategy
+    detection_strategy: FailureDetectionStrategy = FailureDetectionStrategy.CONSECUTIVE
+    failure_rate_threshold: float = 0.5
+    time_window_seconds: float = 60.0
+    min_calls_in_window: int = 10
+
+    # Exception filtering
+    excluded_exceptions: tuple[type[Exception], ...] = ()
+    included_exceptions: tuple[type[Exception], ...] | None = None
+
+    # Callbacks
+    fallback: Callable[..., Any] | None = None
+    on_state_change: Callable[[StateChangeEvent], None] | None = None
+    on_failure: Callable[[Exception, CircuitBreakerMetrics], None] | None = None
+    on_success: Callable[[Any, CircuitBreakerMetrics], None] | None = None
+
+    def should_count_exception(self, exc: Exception) -> bool:
+        """Determine if an exception should count as a failure."""
+        if self.excluded_exceptions and isinstance(exc, self.excluded_exceptions):
+            return False
+        if self.included_exceptions is not None:
+            return isinstance(exc, self.included_exceptions)
+        return True
+
+
+class CircuitBreakerError(Exception):
+    """Base exception for circuit breaker errors."""
+
+    def __init__(
+        self,
+        message: str,
+        breaker_name: str | None = None,
+        state: CircuitState | None = None,
+    ):
+        super().__init__(message)
+        self.breaker_name = breaker_name
+        self.state = state
+
+
+class CircuitOpenError(CircuitBreakerError):
+    """Raised when attempting to call through an open circuit."""
+
+    def __init__(
+        self,
+        breaker_name: str,
+        remaining_time: float | None = None,
+    ):
+        message = f"Circuit '{breaker_name}' is open"
+        if remaining_time is not None:
+            message += f", recovery in {remaining_time:.1f}s"
+        super().__init__(message, breaker_name, CircuitState.OPEN)
+        self.remaining_time = remaining_time
+
+
+class CircuitHalfOpenError(CircuitBreakerError):
+    """Raised when half-open circuit has reached max test calls."""
+
+    def __init__(self, breaker_name: str, max_calls: int):
+        message = f"Circuit '{breaker_name}' is half-open and at max test calls ({max_calls})"
+        super().__init__(message, breaker_name, CircuitState.HALF_OPEN)
+        self.max_calls = max_calls
+
+
+@dataclass
+class CallResult:
+    """Result of a call through the circuit breaker.
+
+    Attributes:
+        success: Whether the call succeeded
+        result: The return value if successful
+        exception: The exception if failed
+        duration_ms: Call duration in milliseconds
+        timestamp: When the call was made
+        from_fallback: Whether result came from fallback
+    """
+
+    success: bool
+    result: Any = None
+    exception: Exception | None = None
+    duration_ms: float = 0.0
+    timestamp: datetime = field(default_factory=datetime.now)
+    from_fallback: bool = False
+
+
+@dataclass
+class CircuitBreakerMetrics:
+    """Metrics for circuit breaker monitoring.
+
+    Attributes:
+        name: Breaker name
+        state: Current state
+        total_calls: Total number of calls
+        successful_calls: Number of successful calls
+        failed_calls: Number of failed calls
+        rejected_calls: Calls rejected due to open circuit
+        consecutive_failures: Current consecutive failure count
+        consecutive_successes: Current consecutive success count
+        failure_rate: Current failure rate (0.0 - 1.0)
+        last_failure_time: Time of last failure
+        last_success_time: Time of last success
+        last_state_change_time: Time of last state change
+        time_in_current_state_ms: Time spent in current state
+        average_response_time_ms: Average call duration
+    """
+
+    name: str
+    state: CircuitState
+    total_calls: int = 0
+    successful_calls: int = 0
+    failed_calls: int = 0
+    rejected_calls: int = 0
+    consecutive_failures: int = 0
+    consecutive_successes: int = 0
+    failure_rate: float = 0.0
+    last_failure_time: datetime | None = None
+    last_success_time: datetime | None = None
+    last_state_change_time: datetime | None = None
+    time_in_current_state_ms: float = 0.0
+    average_response_time_ms: float = 0.0
+
+    @property
+    def is_healthy(self) -> bool:
+        """Check if circuit is in healthy state."""
+        return self.state == CircuitState.CLOSED
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "name": self.name,
+            "state": self.state.value,
+            "total_calls": self.total_calls,
+            "successful_calls": self.successful_calls,
+            "failed_calls": self.failed_calls,
+            "rejected_calls": self.rejected_calls,
+            "consecutive_failures": self.consecutive_failures,
+            "consecutive_successes": self.consecutive_successes,
+            "failure_rate": self.failure_rate,
+            "last_failure_time": self.last_failure_time.isoformat() if self.last_failure_time else None,
+            "last_success_time": self.last_success_time.isoformat() if self.last_success_time else None,
+            "last_state_change_time": self.last_state_change_time.isoformat() if self.last_state_change_time else None,
+            "time_in_current_state_ms": self.time_in_current_state_ms,
+            "average_response_time_ms": self.average_response_time_ms,
+            "is_healthy": self.is_healthy,
+        }
+
+
+@dataclass
+class StateChangeEvent:
+    """Event emitted when circuit state changes.
+
+    Attributes:
+        breaker_name: Name of the circuit breaker
+        from_state: Previous state
+        to_state: New state
+        timestamp: When the change occurred
+        reason: Reason for state change
+        metrics: Metrics at time of change
+    """
+
+    breaker_name: str
+    from_state: CircuitState
+    to_state: CircuitState
+    timestamp: datetime = field(default_factory=datetime.now)
+    reason: str = ""
+    metrics: CircuitBreakerMetrics | None = None
+
+    def __str__(self) -> str:
+        return (
+            f"CircuitBreaker '{self.breaker_name}' state change: "
+            f"{self.from_state.value} -> {self.to_state.value}"
+            f"{f' ({self.reason})' if self.reason else ''}"
+        )

truthound/checkpoint/circuitbreaker/detection.py

@@ -0,0 +1,459 @@
+"""Failure detection strategies for Circuit Breaker.
+
+This module provides different strategies for determining when
+a circuit should trip (open):
+
+- ConsecutiveFailureDetector: Trip after N consecutive failures
+- PercentageFailureDetector: Trip when failure rate exceeds threshold
+- TimeWindowFailureDetector: Trip based on failures within time window
+- CompositeFailureDetector: Combine multiple strategies
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections import deque
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from threading import RLock
+from typing import Protocol, runtime_checkable
+
+from truthound.checkpoint.circuitbreaker.core import (
+    CircuitBreakerConfig,
+    FailureDetectionStrategy,
+)
+
+
+@runtime_checkable
+class FailureDetector(Protocol):
+    """Protocol for failure detection strategies."""
+
+    def record_success(self) -> None:
+        """Record a successful call."""
+        ...
+
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record a failed call."""
+        ...
+
+    def should_trip(self) -> bool:
+        """Determine if circuit should trip (open)."""
+        ...
+
+    def reset(self) -> None:
+        """Reset detector state."""
+        ...
+
+    @property
+    def failure_count(self) -> int:
+        """Current failure count."""
+        ...
+
+    @property
+    def success_count(self) -> int:
+        """Current success count."""
+        ...
+
+
+class BaseFailureDetector(ABC):
+    """Base class for failure detectors with common functionality."""
+
+    def __init__(self):
+        self._lock = RLock()
+        self._failures = 0
+        self._successes = 0
+
+    @abstractmethod
+    def record_success(self) -> None:
+        """Record a successful call."""
+        pass
+
+    @abstractmethod
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record a failed call."""
+        pass
+
+    @abstractmethod
+    def should_trip(self) -> bool:
+        """Determine if circuit should trip."""
+        pass
+
+    def reset(self) -> None:
+        """Reset detector state."""
+        with self._lock:
+            self._failures = 0
+            self._successes = 0
+
+    @property
+    def failure_count(self) -> int:
+        """Current failure count."""
+        return self._failures
+
+    @property
+    def success_count(self) -> int:
+        """Current success count."""
+        return self._successes
+
+
+class ConsecutiveFailureDetector(BaseFailureDetector):
+    """Trips after N consecutive failures.
+
+    This is the simplest detection strategy. The circuit trips when
+    a specified number of consecutive failures occur. A single success
+    resets the counter.
+
+    Example:
+        >>> detector = ConsecutiveFailureDetector(threshold=3)
+        >>> detector.record_failure()  # count: 1
+        >>> detector.record_failure()  # count: 2
+        >>> detector.should_trip()      # False
+        >>> detector.record_failure()  # count: 3
+        >>> detector.should_trip()      # True
+    """
+
+    def __init__(self, threshold: int = 5):
+        """Initialize detector.
+
+        Args:
+            threshold: Number of consecutive failures to trip
+        """
+        super().__init__()
+        self._threshold = threshold
+        self._consecutive_failures = 0
+
+    def record_success(self) -> None:
+        """Record success and reset consecutive failures."""
+        with self._lock:
+            self._successes += 1
+            self._consecutive_failures = 0
+
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record failure and increment consecutive counter."""
+        with self._lock:
+            self._failures += 1
+            self._consecutive_failures += 1
+
+    def should_trip(self) -> bool:
+        """Trip if consecutive failures >= threshold."""
+        with self._lock:
+            return self._consecutive_failures >= self._threshold
+
+    def reset(self) -> None:
+        """Reset all counters."""
+        with self._lock:
+            super().reset()
+            self._consecutive_failures = 0
+
+    @property
+    def consecutive_failures(self) -> int:
+        """Current consecutive failure count."""
+        return self._consecutive_failures
+
+
+class PercentageFailureDetector(BaseFailureDetector):
+    """Trips when failure percentage exceeds threshold.
+
+    This strategy considers the overall failure rate rather than
+    consecutive failures. Useful for services with occasional
+    transient errors.
+
+    Example:
+        >>> detector = PercentageFailureDetector(
+        ...     threshold=0.5,  # 50%
+        ...     min_calls=10,
+        ... )
+        >>> for _ in range(6):
+        ...     detector.record_failure()
+        >>> for _ in range(4):
+        ...     detector.record_success()
+        >>> detector.should_trip()  # True (60% failure rate)
+    """
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        min_calls: int = 10,
+        window_size: int | None = None,
+    ):
+        """Initialize detector.
+
+        Args:
+            threshold: Failure rate (0.0-1.0) to trip
+            min_calls: Minimum calls before percentage applies
+            window_size: If set, only consider last N calls
+        """
+        super().__init__()
+        self._threshold = threshold
+        self._min_calls = min_calls
+        self._window_size = window_size
+        self._calls: deque[bool] = deque(maxlen=window_size)
+
+    def record_success(self) -> None:
+        """Record success."""
+        with self._lock:
+            self._successes += 1
+            self._calls.append(True)
+
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record failure."""
+        with self._lock:
+            self._failures += 1
+            self._calls.append(False)
+
+    def should_trip(self) -> bool:
+        """Trip if failure rate >= threshold and min calls met."""
+        with self._lock:
+            total = len(self._calls) if self._window_size else (self._successes + self._failures)
+            if total < self._min_calls:
+                return False
+
+            if self._window_size:
+                failures = sum(1 for success in self._calls if not success)
+                rate = failures / total
+            else:
+                rate = self._failures / total if total > 0 else 0.0
+
+            return rate >= self._threshold
+
+    def reset(self) -> None:
+        """Reset all counters."""
+        with self._lock:
+            super().reset()
+            self._calls.clear()
+
+    @property
+    def failure_rate(self) -> float:
+        """Current failure rate."""
+        with self._lock:
+            if self._window_size:
+                total = len(self._calls)
+                if total == 0:
+                    return 0.0
+                failures = sum(1 for success in self._calls if not success)
+                return failures / total
+            else:
+                total = self._successes + self._failures
+                return self._failures / total if total > 0 else 0.0
+
+
+@dataclass
+class TimestampedCall:
+    """A call with timestamp for time-window tracking."""
+
+    success: bool
+    timestamp: datetime = field(default_factory=datetime.now)
+    exception_type: str | None = None
+
+
+class TimeWindowFailureDetector(BaseFailureDetector):
+    """Trips based on failures within a sliding time window.
+
+    This strategy only considers recent calls, allowing the circuit
+    to recover naturally as old failures age out of the window.
+
+    Example:
+        >>> detector = TimeWindowFailureDetector(
+        ...     threshold=5,
+        ...     window_seconds=60.0,
+        ... )
+        >>> # 5 failures within 60 seconds will trip
+    """
+
+    def __init__(
+        self,
+        threshold: int = 5,
+        window_seconds: float = 60.0,
+        use_percentage: bool = False,
+        percentage_threshold: float = 0.5,
+        min_calls: int = 10,
+    ):
+        """Initialize detector.
+
+        Args:
+            threshold: Number of failures to trip (absolute mode)
+            window_seconds: Time window size in seconds
+            use_percentage: Use percentage mode instead of absolute
+            percentage_threshold: Failure rate to trip (percentage mode)
+            min_calls: Minimum calls for percentage mode
+        """
+        super().__init__()
+        self._threshold = threshold
+        self._window_seconds = window_seconds
+        self._use_percentage = use_percentage
+        self._percentage_threshold = percentage_threshold
+        self._min_calls = min_calls
+        self._calls: list[TimestampedCall] = []
+
+    def _cleanup_old_calls(self) -> None:
+        """Remove calls outside the time window."""
+        cutoff = datetime.now() - timedelta(seconds=self._window_seconds)
+        self._calls = [c for c in self._calls if c.timestamp >= cutoff]
+
+    def record_success(self) -> None:
+        """Record success."""
+        with self._lock:
+            self._successes += 1
+            self._calls.append(TimestampedCall(success=True))
+            self._cleanup_old_calls()
+
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record failure."""
+        with self._lock:
+            self._failures += 1
+            exc_type = type(exception).__name__ if exception else None
+            self._calls.append(TimestampedCall(success=False, exception_type=exc_type))
+            self._cleanup_old_calls()
+
+    def should_trip(self) -> bool:
+        """Trip based on failures in time window."""
+        with self._lock:
+            self._cleanup_old_calls()
+
+            failures_in_window = sum(1 for c in self._calls if not c.success)
+            total_in_window = len(self._calls)
+
+            if self._use_percentage:
+                if total_in_window < self._min_calls:
+                    return False
+                rate = failures_in_window / total_in_window
+                return rate >= self._percentage_threshold
+            else:
+                return failures_in_window >= self._threshold
+
+    def reset(self) -> None:
+        """Reset all counters."""
+        with self._lock:
+            super().reset()
+            self._calls.clear()
+
+    @property
+    def calls_in_window(self) -> int:
+        """Number of calls in current window."""
+        with self._lock:
+            self._cleanup_old_calls()
+            return len(self._calls)
+
+    @property
+    def failures_in_window(self) -> int:
+        """Number of failures in current window."""
+        with self._lock:
+            self._cleanup_old_calls()
+            return sum(1 for c in self._calls if not c.success)
+
+
+class CompositeFailureDetector(BaseFailureDetector):
+    """Combines multiple detection strategies.
+
+    This detector can use either AND or OR logic to combine
+    multiple detection strategies.
+
+    Example:
+        >>> detector = CompositeFailureDetector(
+        ...     detectors=[
+        ...         ConsecutiveFailureDetector(threshold=3),
+        ...         PercentageFailureDetector(threshold=0.5),
+        ...     ],
+        ...     require_all=False,  # OR logic
+        ... )
+    """
+
+    def __init__(
+        self,
+        detectors: list[FailureDetector],
+        require_all: bool = False,
+    ):
+        """Initialize composite detector.
+
+        Args:
+            detectors: List of detectors to combine
+            require_all: If True, all must trip (AND). If False, any trips (OR)
+        """
+        super().__init__()
+        self._detectors = detectors
+        self._require_all = require_all
+
+    def record_success(self) -> None:
+        """Record success on all detectors."""
+        with self._lock:
+            self._successes += 1
+            for detector in self._detectors:
+                detector.record_success()
+
+    def record_failure(self, exception: Exception | None = None) -> None:
+        """Record failure on all detectors."""
+        with self._lock:
+            self._failures += 1
+            for detector in self._detectors:
+                detector.record_failure(exception)
+
+    def should_trip(self) -> bool:
+        """Trip based on combined detector results."""
+        with self._lock:
+            if self._require_all:
+                return all(d.should_trip() for d in self._detectors)
+            else:
+                return any(d.should_trip() for d in self._detectors)
+
+    def reset(self) -> None:
+        """Reset all detectors."""
+        with self._lock:
+            super().reset()
+            for detector in self._detectors:
+                detector.reset()
+
+    @property
+    def detector_states(self) -> list[dict]:
+        """Get state of all detectors."""
+        return [
+            {
+                "type": type(d).__name__,
+                "should_trip": d.should_trip(),
+                "failure_count": d.failure_count,
+                "success_count": d.success_count,
+            }
+            for d in self._detectors
+        ]
+
+
+def create_detector(config: CircuitBreakerConfig) -> FailureDetector:
+    """Factory function to create appropriate detector from config.
+
+    Args:
+        config: Circuit breaker configuration
+
+    Returns:
+        Appropriate failure detector based on config.detection_strategy
+    """
+    strategy = config.detection_strategy
+
+    if strategy == FailureDetectionStrategy.CONSECUTIVE:
+        return ConsecutiveFailureDetector(threshold=config.failure_threshold)
+
+    elif strategy == FailureDetectionStrategy.PERCENTAGE:
+        return PercentageFailureDetector(
+            threshold=config.failure_rate_threshold,
+            min_calls=config.min_calls_in_window,
+        )
+
+    elif strategy == FailureDetectionStrategy.TIME_WINDOW:
+        return TimeWindowFailureDetector(
+            threshold=config.failure_threshold,
+            window_seconds=config.time_window_seconds,
+        )
+
+    elif strategy == FailureDetectionStrategy.COMPOSITE:
+        # Create a composite with both consecutive and percentage
+        return CompositeFailureDetector(
+            detectors=[
+                ConsecutiveFailureDetector(threshold=config.failure_threshold),
+                PercentageFailureDetector(
+                    threshold=config.failure_rate_threshold,
+                    min_calls=config.min_calls_in_window,
+                ),
+            ],
+            require_all=False,
+        )
+
+    else:
+        # Default to consecutive
+        return ConsecutiveFailureDetector(threshold=config.failure_threshold)