dory-sdk 2.1.0__py3-none-any.whl

dory/recovery/recovery_decision.py

@@ -0,0 +1,242 @@
+"""
+Recovery decision maker.
+
+Determines the appropriate recovery strategy based on
+restart count, failure type, and migration status.
+"""
+
+import logging
+from dataclasses import dataclass
+from enum import Enum
+
+from dory.types import RecoveryStrategy, FaultType
+
+logger = logging.getLogger(__name__)
+
+
+class DecisionReason(Enum):
+    """Reasons for recovery decisions."""
+    FIRST_START = "first_start"
+    MIGRATION = "migration"
+    NORMAL_RESTART = "normal_restart"
+    RAPID_RESTART = "rapid_restart"
+    THRESHOLD_EXCEEDED = "threshold_exceeded"
+    STATE_CORRUPTION = "state_corruption"
+    CRASH_LOOP = "crash_loop"
+
+
+@dataclass
+class RecoveryDecision:
+    """Result of recovery decision making."""
+    strategy: RecoveryStrategy
+    reason: DecisionReason
+    should_restore_state: bool
+    should_clear_caches: bool
+    backoff_seconds: int = 0
+    message: str = ""
+
+    @property
+    def name(self) -> str:
+        """Get strategy name for logging."""
+        return self.strategy.value
+
+
+class RecoveryDecisionMaker:
+    """
+    Decides recovery strategy based on context.
+
+    Strategies:
+    1. RESTORE_STATE - Normal recovery, restore from checkpoint
+    2. GOLDEN_IMAGE - Full reset, start fresh
+    3. GOLDEN_WITH_BACKOFF - Reset with delay to prevent rapid cycling
+    """
+
+    def __init__(
+        self,
+        golden_image_threshold: int = 3,
+        rapid_restart_window_sec: int = 60,
+        max_backoff_sec: int = 300,
+    ):
+        """
+        Initialize decision maker.
+
+        Args:
+            golden_image_threshold: Restart count triggering golden image
+            rapid_restart_window_sec: Window for detecting rapid restarts
+            max_backoff_sec: Maximum backoff delay
+        """
+        self._golden_threshold = golden_image_threshold
+        self._rapid_window = rapid_restart_window_sec
+        self._max_backoff = max_backoff_sec
+
+    def decide(
+        self,
+        restart_count: int,
+        is_migrating: bool = False,
+        fault_type: FaultType | None = None,
+        state_valid: bool = True,
+        state_exists: bool = False,
+    ) -> RecoveryDecision:
+        """
+        Decide recovery strategy.
+
+        Args:
+            restart_count: Current restart count
+            is_migrating: Whether this is a migration restart
+            fault_type: Type of fault that caused restart
+            state_valid: Whether existing state is valid
+            state_exists: Whether saved state exists (e.g., in ConfigMap)
+
+        Returns:
+            RecoveryDecision with strategy and details
+        """
+        # First start - but if state exists, it means this is a pod replacement
+        # (orchestrator created a new pod after deleting the old one)
+        if restart_count == 0:
+            if state_exists:
+                # State exists from previous pod - treat as migration/replacement
+                return RecoveryDecision(
+                    strategy=RecoveryStrategy.RESTORE_STATE,
+                    reason=DecisionReason.MIGRATION,
+                    should_restore_state=True,
+                    should_clear_caches=False,
+                    message="Pod replacement detected (state exists), restoring state",
+                )
+            # Truly first start with no prior state
+            return RecoveryDecision(
+                strategy=RecoveryStrategy.RESTORE_STATE,
+                reason=DecisionReason.FIRST_START,
+                should_restore_state=False,
+                should_clear_caches=False,
+                message="First start, no state to restore",
+            )
+
+        # Migration - always restore state
+        if is_migrating:
+            return RecoveryDecision(
+                strategy=RecoveryStrategy.RESTORE_STATE,
+                reason=DecisionReason.MIGRATION,
+                should_restore_state=True,
+                should_clear_caches=False,
+                message="Migration restart, restoring state",
+            )
+
+        # State corruption - golden image reset
+        if not state_valid or fault_type == FaultType.STATE_CORRUPTION:
+            return RecoveryDecision(
+                strategy=RecoveryStrategy.GOLDEN_IMAGE,
+                reason=DecisionReason.STATE_CORRUPTION,
+                should_restore_state=False,
+                should_clear_caches=True,
+                message="State corruption detected, performing golden image reset",
+            )
+
+        # Threshold exceeded - golden image with backoff
+        if restart_count >= self._golden_threshold:
+            backoff = self._calculate_backoff(restart_count)
+            return RecoveryDecision(
+                strategy=RecoveryStrategy.GOLDEN_WITH_BACKOFF,
+                reason=DecisionReason.THRESHOLD_EXCEEDED,
+                should_restore_state=False,
+                should_clear_caches=True,
+                backoff_seconds=backoff,
+                message=f"Restart threshold exceeded ({restart_count} >= {self._golden_threshold}), "
+                        f"golden image reset with {backoff}s backoff",
+            )
+
+        # Normal restart - try to restore state
+        return RecoveryDecision(
+            strategy=RecoveryStrategy.RESTORE_STATE,
+            reason=DecisionReason.NORMAL_RESTART,
+            should_restore_state=True,
+            should_clear_caches=True,
+            message=f"Normal restart (attempt {restart_count + 1}), restoring state",
+        )
+
+    def _calculate_backoff(self, restart_count: int) -> int:
+        """
+        Calculate backoff delay based on restart count.
+
+        Uses exponential backoff with jitter.
+        """
+        base_backoff = 10  # seconds
+        # Exponential: 10, 20, 40, 80, 160, ... capped at max
+        backoff = min(
+            base_backoff * (2 ** (restart_count - self._golden_threshold)),
+            self._max_backoff,
+        )
+        return int(backoff)
+
+    def should_trigger_alert(self, restart_count: int) -> bool:
+        """
+        Check if restart count should trigger alerting.
+
+        Args:
+            restart_count: Current restart count
+
+        Returns:
+            True if alert should be triggered
+        """
+        # Alert on first golden image reset and every N restarts after
+        if restart_count == self._golden_threshold:
+            return True
+        if restart_count > self._golden_threshold and restart_count % 3 == 0:
+            return True
+        return False
+
+
+class RecoveryExecutor:
+    """
+    Executes recovery decisions.
+
+    Coordinates the actual recovery steps based on decision.
+    """
+
+    def __init__(self, state_manager, golden_image_manager):
+        """
+        Initialize recovery executor.
+
+        Args:
+            state_manager: State manager for state operations
+            golden_image_manager: Golden image manager for resets
+        """
+        self._state_manager = state_manager
+        self._golden_manager = golden_image_manager
+
+    async def execute(
+        self,
+        decision: RecoveryDecision,
+        processor_id: str,
+    ) -> dict | None:
+        """
+        Execute recovery decision.
+
+        Args:
+            decision: Recovery decision to execute
+            processor_id: Processor ID
+
+        Returns:
+            Restored state dict, or None if golden image reset
+        """
+        logger.info(f"Executing recovery: {decision.strategy.value} - {decision.message}")
+
+        # Apply backoff if needed
+        if decision.backoff_seconds > 0:
+            logger.info(f"Applying backoff: {decision.backoff_seconds}s")
+            import asyncio
+            await asyncio.sleep(decision.backoff_seconds)
+
+        # Golden image reset
+        if decision.strategy in (
+            RecoveryStrategy.GOLDEN_IMAGE,
+            RecoveryStrategy.GOLDEN_WITH_BACKOFF,
+        ):
+            await self._golden_manager.reset(processor_id)
+            return None
+
+        # Restore state
+        if decision.should_restore_state:
+            state = await self._state_manager.load_state(processor_id)
+            return state
+
+        return None

dory/recovery/restart_detector.py

@@ -0,0 +1,142 @@
+"""
+Restart detection for pod lifecycle tracking.
+
+Detects restarts by checking restart count from:
+1. Kubernetes downward API (restart count annotation)
+2. Local file marker
+3. Environment variables
+"""
+
+import logging
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RestartInfo:
+    """Information about restart status."""
+    restart_count: int
+    is_restart: bool
+    previous_exit_code: int | None = None
+    restart_reason: str | None = None
+
+    @property
+    def is_first_start(self) -> bool:
+        """Check if this is the first start (not a restart)."""
+        return not self.is_restart
+
+
+class RestartDetector:
+    """
+    Detects pod restarts and tracks restart count.
+
+    Uses multiple methods to detect restarts:
+    1. RESTART_COUNT environment variable (set by init container)
+    2. Local marker file with count
+    3. Kubernetes pod annotation via downward API
+    """
+
+    MARKER_FILE_PATH = "/tmp/dory-restart-marker"
+    RESTART_COUNT_ENV = "RESTART_COUNT"
+    PREVIOUS_EXIT_CODE_ENV = "PREVIOUS_EXIT_CODE"
+    RESTART_REASON_ENV = "RESTART_REASON"
+
+    def __init__(self, marker_path: str | None = None):
+        """
+        Initialize restart detector.
+
+        Args:
+            marker_path: Optional custom path for marker file
+        """
+        self._marker_path = Path(marker_path or self.MARKER_FILE_PATH)
+
+    async def detect(self) -> RestartInfo:
+        """
+        Detect restart status.
+
+        Returns:
+            RestartInfo with restart count and status
+        """
+        # Try environment variable first (most reliable in K8s)
+        env_count = self._detect_from_env()
+        if env_count is not None:
+            logger.debug(f"Restart count from env: {env_count}")
+            return RestartInfo(
+                restart_count=env_count,
+                is_restart=env_count > 0,
+                previous_exit_code=self._get_previous_exit_code(),
+                restart_reason=self._get_restart_reason(),
+            )
+
+        # Fall back to marker file
+        marker_count = self._detect_from_marker()
+        logger.debug(f"Restart count from marker: {marker_count}")
+
+        # Increment and save marker for next restart
+        self._save_marker(marker_count + 1)
+
+        return RestartInfo(
+            restart_count=marker_count,
+            is_restart=marker_count > 0,
+            previous_exit_code=self._get_previous_exit_code(),
+            restart_reason=self._get_restart_reason(),
+        )
+
+    def _detect_from_env(self) -> int | None:
+        """Detect restart count from environment variable."""
+        count_str = os.environ.get(self.RESTART_COUNT_ENV)
+        if count_str is None:
+            return None
+
+        try:
+            return int(count_str)
+        except ValueError:
+            logger.warning(f"Invalid RESTART_COUNT value: {count_str}")
+            return None
+
+    def _detect_from_marker(self) -> int:
+        """Detect restart count from marker file."""
+        if not self._marker_path.exists():
+            return 0
+
+        try:
+            content = self._marker_path.read_text().strip()
+            return int(content)
+        except (ValueError, IOError) as e:
+            logger.warning(f"Failed to read marker file: {e}")
+            return 0
+
+    def _save_marker(self, count: int) -> None:
+        """Save restart count to marker file."""
+        try:
+            self._marker_path.parent.mkdir(parents=True, exist_ok=True)
+            self._marker_path.write_text(str(count))
+        except IOError as e:
+            logger.warning(f"Failed to save marker file: {e}")
+
+    def _get_previous_exit_code(self) -> int | None:
+        """Get previous exit code from environment."""
+        code_str = os.environ.get(self.PREVIOUS_EXIT_CODE_ENV)
+        if code_str is None:
+            return None
+
+        try:
+            return int(code_str)
+        except ValueError:
+            return None
+
+    def _get_restart_reason(self) -> str | None:
+        """Get restart reason from environment."""
+        return os.environ.get(self.RESTART_REASON_ENV)
+
+    def reset(self) -> None:
+        """Reset restart counter (for testing or golden image reset)."""
+        if self._marker_path.exists():
+            try:
+                self._marker_path.unlink()
+                logger.info("Restart marker reset")
+            except IOError as e:
+                logger.warning(f"Failed to reset marker: {e}")

dory/recovery/state_validator.py

@@ -0,0 +1,187 @@
+"""
+State validation for integrity checking.
+
+Validates restored state against schema and checksums.
+"""
+
+import logging
+from typing import Any
+
+from dory.utils.errors import DoryValidationError
+
+logger = logging.getLogger(__name__)
+
+
+class StateValidator:
+    """
+    Validates processor state for integrity and schema compliance.
+
+    Performs:
+    1. Schema validation (required fields, types)
+    2. Integrity checks (checksums)
+    3. Version compatibility checks
+    """
+
+    def __init__(self, schema: dict[str, type] | None = None):
+        """
+        Initialize validator.
+
+        Args:
+            schema: Optional schema mapping field names to expected types
+        """
+        self._schema = schema
+
+    def validate(self, state: dict[str, Any]) -> bool:
+        """
+        Validate state dictionary.
+
+        Args:
+            state: State dictionary to validate
+
+        Returns:
+            True if valid
+
+        Raises:
+            DoryValidationError: If validation fails
+        """
+        if not isinstance(state, dict):
+            raise DoryValidationError(f"State must be a dict, got {type(state)}")
+
+        # Validate against schema if provided
+        if self._schema:
+            self._validate_schema(state)
+
+        # Run integrity checks
+        self._validate_integrity(state)
+
+        logger.debug("State validation passed")
+        return True
+
+    def _validate_schema(self, state: dict[str, Any]) -> None:
+        """Validate state against schema."""
+        for field_name, expected_type in self._schema.items():
+            if field_name not in state:
+                raise DoryValidationError(
+                    f"Required field '{field_name}' missing from state"
+                )
+
+            value = state[field_name]
+
+            # Allow None for any type
+            if value is None:
+                continue
+
+            if not isinstance(value, expected_type):
+                raise DoryValidationError(
+                    f"Field '{field_name}' has wrong type: "
+                    f"expected {expected_type.__name__}, got {type(value).__name__}"
+                )
+
+    def _validate_integrity(self, state: dict[str, Any]) -> None:
+        """
+        Run integrity checks on state.
+
+        Can be extended for custom integrity validation.
+        """
+        # Check for common corruption indicators
+        if "__corrupted__" in state:
+            raise DoryValidationError("State marked as corrupted")
+
+        # Check metadata if present
+        if "_metadata" in state:
+            metadata = state["_metadata"]
+            if not isinstance(metadata, dict):
+                raise DoryValidationError("State metadata must be a dict")
+
+    def validate_partial(self, state: dict[str, Any], required_fields: list[str]) -> bool:
+        """
+        Validate that specific fields exist and have correct types.
+
+        Args:
+            state: State dictionary
+            required_fields: List of field names that must exist
+
+        Returns:
+            True if valid
+
+        Raises:
+            DoryValidationError: If validation fails
+        """
+        for field_name in required_fields:
+            if field_name not in state:
+                raise DoryValidationError(
+                    f"Required field '{field_name}' missing from state"
+                )
+
+            if self._schema and field_name in self._schema:
+                expected_type = self._schema[field_name]
+                value = state[field_name]
+
+                if value is not None and not isinstance(value, expected_type):
+                    raise DoryValidationError(
+                        f"Field '{field_name}' has wrong type"
+                    )
+
+        return True
+
+
+class StateVersionChecker:
+    """
+    Checks state version compatibility.
+
+    Ensures restored state is compatible with current processor version.
+    """
+
+    VERSION_FIELD = "_version"
+
+    def __init__(self, current_version: str):
+        """
+        Initialize version checker.
+
+        Args:
+            current_version: Current processor state version
+        """
+        self._current_version = current_version
+
+    def check_compatible(self, state: dict[str, Any]) -> bool:
+        """
+        Check if state version is compatible.
+
+        Args:
+            state: State dictionary
+
+        Returns:
+            True if compatible
+
+        Raises:
+            DoryValidationError: If incompatible
+        """
+        state_version = state.get(self.VERSION_FIELD)
+
+        if state_version is None:
+            # No version = assume compatible (v0)
+            logger.warning("State has no version field, assuming compatible")
+            return True
+
+        if not self._is_compatible(state_version, self._current_version):
+            raise DoryValidationError(
+                f"State version {state_version} not compatible "
+                f"with processor version {self._current_version}"
+            )
+
+        return True
+
+    def _is_compatible(self, state_version: str, processor_version: str) -> bool:
+        """
+        Check version compatibility.
+
+        Default: major version must match.
+        Override for custom compatibility logic.
+        """
+        try:
+            state_major = int(state_version.split(".")[0])
+            processor_major = int(processor_version.split(".")[0])
+            return state_major == processor_major
+        except (ValueError, IndexError):
+            # Invalid version format
+            return False

dory/resilience/__init__.py

@@ -0,0 +1,45 @@
+"""
+Resilience patterns for fault-tolerant processing.
+
+This module provides production-ready resilience patterns:
+- Retry with exponential backoff
+- Circuit breaker pattern
+- Rate limiting
+- Bulkhead isolation
+
+Example usage:
+    from dory.resilience import retry_with_backoff, CircuitBreaker
+
+    @retry_with_backoff(max_attempts=3)
+    async def call_api():
+        return await api.get()
+
+    breaker = CircuitBreaker(name="database")
+    result = await breaker.call(db.query)
+"""
+
+from .retry import (
+    retry_with_backoff,
+    RetryPolicy,
+    RetryBudget,
+    RetryExhaustedError,
+)
+from .circuit_breaker import (
+    CircuitBreaker,
+    CircuitState,
+    CircuitOpenError,
+    CircuitBreakerConfig,
+)
+
+__all__ = [
+    # Retry
+    "retry_with_backoff",
+    "RetryPolicy",
+    "RetryBudget",
+    "RetryExhaustedError",
+    # Circuit Breaker
+    "CircuitBreaker",
+    "CircuitState",
+    "CircuitOpenError",
+    "CircuitBreakerConfig",
+]