mcp-hangar 0.2.0__py3-none-any.whl

mcp_hangar/application/sagas/provider_failover_saga.py

@@ -0,0 +1,266 @@
+"""Provider Failover Saga - failover to backup providers on failure."""
+
+from dataclasses import dataclass
+import time
+from typing import Dict, List, Optional, Set, Type
+
+from ...domain.events import DomainEvent, ProviderDegraded, ProviderStarted, ProviderStopped
+from ...infrastructure.saga_manager import EventTriggeredSaga
+from ...logging_config import get_logger
+from ..commands import Command, StartProviderCommand, StopProviderCommand
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class FailoverConfig:
+    """Configuration for a failover pair."""
+
+    primary_id: str
+    backup_id: str
+    auto_failback: bool = True  # Automatically fail back to primary when it recovers
+    failback_delay_s: float = 30.0  # Delay before failing back to primary
+
+
+@dataclass
+class FailoverState:
+    """State of an active failover."""
+
+    primary_id: str
+    backup_id: str
+    failed_at: float
+    backup_started_at: Optional[float] = None
+    is_active: bool = True
+
+
+class ProviderFailoverSaga(EventTriggeredSaga):
+    """
+    Saga that orchestrates failover to backup providers.
+
+    Failover Strategy:
+    1. Configure primary-backup pairs
+    2. When primary is degraded/stopped, start the backup
+    3. Optionally, fail back to primary when it recovers
+    4. Track active failovers to prevent cycles
+
+    Configuration:
+    - Failover pairs: Define which providers are backups for others
+    - Auto-failback: Whether to automatically switch back to primary
+    - Failback delay: How long to wait before failing back
+
+    Usage:
+        saga = ProviderFailoverSaga()
+        saga.configure_failover("primary-provider", "backup-provider")
+        saga_manager.register_event_saga(saga)
+    """
+
+    def __init__(self):
+        super().__init__()
+
+        # Failover configuration: primary_id -> FailoverConfig
+        self._failover_configs: Dict[str, FailoverConfig] = {}
+
+        # Active failovers: primary_id -> FailoverState
+        self._active_failovers: Dict[str, FailoverState] = {}
+
+        # Providers currently acting as backups (to avoid cascading failovers)
+        self._active_backups: Set[str] = set()
+
+        # Providers pending failback: primary_id -> scheduled_time
+        self._pending_failbacks: Dict[str, float] = {}
+
+    @property
+    def saga_type(self) -> str:
+        return "provider_failover"
+
+    @property
+    def handled_events(self) -> List[Type[DomainEvent]]:
+        return [ProviderDegraded, ProviderStarted, ProviderStopped]
+
+    def configure_failover(
+        self,
+        primary_id: str,
+        backup_id: str,
+        auto_failback: bool = True,
+        failback_delay_s: float = 30.0,
+    ) -> None:
+        """
+        Configure a failover pair.
+
+        Args:
+            primary_id: Primary provider ID
+            backup_id: Backup provider ID
+            auto_failback: Whether to automatically fail back when primary recovers
+            failback_delay_s: Delay before failing back
+        """
+        self._failover_configs[primary_id] = FailoverConfig(
+            primary_id=primary_id,
+            backup_id=backup_id,
+            auto_failback=auto_failback,
+            failback_delay_s=failback_delay_s,
+        )
+        logger.info(f"Configured failover: {primary_id} -> {backup_id}")
+
+    def remove_failover(self, primary_id: str) -> bool:
+        """Remove a failover configuration."""
+        if primary_id in self._failover_configs:
+            del self._failover_configs[primary_id]
+            return True
+        return False
+
+    def handle(self, event: DomainEvent) -> List[Command]:
+        """Handle failover-related events."""
+        if isinstance(event, ProviderDegraded):
+            return self._handle_degraded(event)
+        elif isinstance(event, ProviderStarted):
+            return self._handle_started(event)
+        elif isinstance(event, ProviderStopped):
+            return self._handle_stopped(event)
+        return []
+
+    def _handle_degraded(self, event: ProviderDegraded) -> List[Command]:
+        """
+        Handle provider degraded event.
+
+        Initiates failover if this is a primary provider with a configured backup.
+        """
+        provider_id = event.provider_id
+        commands = []
+
+        # Check if this provider is a backup currently serving
+        if provider_id in self._active_backups:
+            logger.warning(f"Backup provider {provider_id} degraded - no further failover")
+            return []
+
+        # Check if this provider has a backup configured
+        config = self._failover_configs.get(provider_id)
+        if not config:
+            return []
+
+        # Check if failover is already active
+        if provider_id in self._active_failovers:
+            logger.debug(f"Failover already active for {provider_id}")
+            return []
+
+        # Initiate failover
+        logger.info(f"Initiating failover: {provider_id} -> {config.backup_id}")
+
+        self._active_failovers[provider_id] = FailoverState(
+            primary_id=provider_id,
+            backup_id=config.backup_id,
+            failed_at=time.time(),
+        )
+        self._active_backups.add(config.backup_id)
+
+        # Start backup provider
+        commands.append(StartProviderCommand(provider_id=config.backup_id))
+
+        return commands
+
+    def _handle_started(self, event: ProviderStarted) -> List[Command]:
+        """
+        Handle provider started event.
+
+        - If it's a backup being started, mark failover as complete
+        - If it's a primary recovering, consider failback
+        """
+        provider_id = event.provider_id
+        commands = []
+
+        # Check if this is a backup being started for failover
+        for primary_id, state in self._active_failovers.items():
+            if state.backup_id == provider_id and state.backup_started_at is None:
+                state.backup_started_at = time.time()
+                logger.info(f"Failover complete: {primary_id} -> {provider_id}")
+
+        # Check if this is a primary recovering
+        if provider_id in self._active_failovers:
+            state = self._active_failovers[provider_id]
+            config = self._failover_configs.get(provider_id)
+
+            if config and config.auto_failback:
+                # Schedule failback
+                failback_time = time.time() + config.failback_delay_s
+                self._pending_failbacks[provider_id] = failback_time
+
+                logger.info(f"Primary {provider_id} recovered, scheduling failback in {config.failback_delay_s}s")
+
+                # In a real implementation, you'd use a scheduler
+                # For now, immediately trigger failback commands
+                commands.extend(self._execute_failback(provider_id))
+
+        return commands
+
+    def _handle_stopped(self, event: ProviderStopped) -> List[Command]:
+        """
+        Handle provider stopped event.
+
+        Clean up failover state if a backup is stopped.
+        """
+        provider_id = event.provider_id
+        commands = []
+
+        # If a backup is stopped, clean up
+        if provider_id in self._active_backups:
+            self._active_backups.discard(provider_id)
+
+            # Find and clean up the failover state
+            for primary_id, state in list(self._active_failovers.items()):
+                if state.backup_id == provider_id:
+                    del self._active_failovers[primary_id]
+                    self._pending_failbacks.pop(primary_id, None)
+                    logger.info(f"Failover {primary_id} -> {provider_id} ended")
+
+        return commands
+
+    def _execute_failback(self, primary_id: str) -> List[Command]:
+        """Execute failback to primary provider."""
+        commands = []
+
+        state = self._active_failovers.get(primary_id)
+        config = self._failover_configs.get(primary_id)
+
+        if not state or not config:
+            return []
+
+        logger.info(f"Executing failback: {state.backup_id} -> {primary_id}")
+
+        # Stop the backup (primary is already running)
+        commands.append(StopProviderCommand(provider_id=state.backup_id, reason="failback"))
+
+        # Clean up failover state
+        del self._active_failovers[primary_id]
+        self._active_backups.discard(state.backup_id)
+        self._pending_failbacks.pop(primary_id, None)
+
+        return commands
+
+    def get_active_failovers(self) -> Dict[str, FailoverState]:
+        """Get all active failovers."""
+        return dict(self._active_failovers)
+
+    def get_failover_config(self, primary_id: str) -> Optional[FailoverConfig]:
+        """Get failover configuration for a provider."""
+        return self._failover_configs.get(primary_id)
+
+    def get_all_configs(self) -> Dict[str, FailoverConfig]:
+        """Get all failover configurations."""
+        return dict(self._failover_configs)
+
+    def is_backup_active(self, provider_id: str) -> bool:
+        """Check if a provider is currently serving as a backup."""
+        return provider_id in self._active_backups
+
+    def force_failback(self, primary_id: str) -> List[Command]:
+        """Manually force a failback to primary."""
+        return self._execute_failback(primary_id)
+
+    def cancel_failover(self, primary_id: str) -> bool:
+        """Cancel an active failover (keeps backup running)."""
+        if primary_id in self._active_failovers:
+            state = self._active_failovers[primary_id]
+            self._active_backups.discard(state.backup_id)
+            del self._active_failovers[primary_id]
+            self._pending_failbacks.pop(primary_id, None)
+            return True
+        return False

mcp_hangar/application/sagas/provider_recovery_saga.py

@@ -0,0 +1,172 @@
+"""Provider Recovery Saga - automatically recover degraded providers."""
+
+import time
+from typing import Dict, List, Optional, Type
+
+from ...domain.events import DomainEvent, HealthCheckFailed, ProviderDegraded, ProviderStarted, ProviderStopped
+from ...infrastructure.saga_manager import EventTriggeredSaga
+from ...logging_config import get_logger
+from ..commands import Command, StartProviderCommand, StopProviderCommand
+
+logger = get_logger(__name__)
+
+
+class ProviderRecoverySaga(EventTriggeredSaga):
+    """
+    Saga that orchestrates automatic provider recovery after failures.
+
+    Recovery Strategy:
+    1. When a provider is degraded, schedule a retry
+    2. Apply exponential backoff between retries
+    3. After max retries, give up and stop the provider
+    4. Reset retry count when provider starts successfully
+
+    Configuration:
+    - max_retries: Maximum number of restart attempts (default: 3)
+    - initial_backoff_s: Initial backoff duration in seconds (default: 5)
+    - max_backoff_s: Maximum backoff duration (default: 60)
+    - backoff_multiplier: Backoff multiplier for exponential growth (default: 2)
+    """
+
+    def __init__(
+        self,
+        max_retries: int = 3,
+        initial_backoff_s: float = 5.0,
+        max_backoff_s: float = 60.0,
+        backoff_multiplier: float = 2.0,
+    ):
+        super().__init__()
+
+        self._max_retries = max_retries
+        self._initial_backoff_s = initial_backoff_s
+        self._max_backoff_s = max_backoff_s
+        self._backoff_multiplier = backoff_multiplier
+
+        # Track retry state per provider
+        # provider_id -> {"retries": int, "last_attempt": float, "next_retry": float}
+        self._retry_state: Dict[str, Dict] = {}
+
+    @property
+    def saga_type(self) -> str:
+        return "provider_recovery"
+
+    @property
+    def handled_events(self) -> List[Type[DomainEvent]]:
+        return [ProviderDegraded, ProviderStarted, ProviderStopped, HealthCheckFailed]
+
+    def handle(self, event: DomainEvent) -> List[Command]:
+        """Handle recovery-related events."""
+        if isinstance(event, ProviderDegraded):
+            return self._handle_degraded(event)
+        elif isinstance(event, ProviderStarted):
+            return self._handle_started(event)
+        elif isinstance(event, ProviderStopped):
+            return self._handle_stopped(event)
+        elif isinstance(event, HealthCheckFailed):
+            return self._handle_health_failed(event)
+        return []
+
+    def _handle_degraded(self, event: ProviderDegraded) -> List[Command]:
+        """
+        Handle provider degraded event.
+
+        Initiates recovery by scheduling a restart with backoff.
+        """
+        provider_id = event.provider_id
+
+        # Initialize retry state if needed
+        if provider_id not in self._retry_state:
+            self._retry_state[provider_id] = {
+                "retries": 0,
+                "last_attempt": 0,
+                "next_retry": 0,
+            }
+
+        state = self._retry_state[provider_id]
+        state["retries"] += 1
+        state["last_attempt"] = time.time()
+
+        # Check if max retries exceeded
+        if state["retries"] > self._max_retries:
+            logger.warning(f"Provider {provider_id} exceeded max retries ({self._max_retries}), stopping recovery")
+            # Stop the provider permanently
+            return [StopProviderCommand(provider_id=provider_id, reason="max_retries_exceeded")]
+
+        # Calculate backoff
+        backoff = self._calculate_backoff(state["retries"])
+        state["next_retry"] = time.time() + backoff
+
+        logger.info(
+            f"Provider {provider_id} degraded, scheduling retry "
+            f"{state['retries']}/{self._max_retries} in {backoff:.1f}s"
+        )
+
+        # Note: In a real implementation, you would use a scheduler
+        # to delay the command. For now, we return it immediately.
+        # The provider's internal backoff will handle timing.
+        return [StartProviderCommand(provider_id=provider_id)]
+
+    def _handle_started(self, event: ProviderStarted) -> List[Command]:
+        """
+        Handle provider started event.
+
+        Resets retry count on successful start.
+        """
+        provider_id = event.provider_id
+
+        if provider_id in self._retry_state:
+            old_retries = self._retry_state[provider_id]["retries"]
+            self._retry_state[provider_id] = {
+                "retries": 0,
+                "last_attempt": 0,
+                "next_retry": 0,
+            }
+            if old_retries > 0:
+                logger.info(f"Provider {provider_id} recovered successfully after {old_retries} retries")
+
+        return []
+
+    def _handle_stopped(self, event: ProviderStopped) -> List[Command]:
+        """
+        Handle provider stopped event.
+
+        Clears retry state for normally stopped providers.
+        """
+        provider_id = event.provider_id
+
+        # Only clear state for intentional stops
+        if event.reason in ("shutdown", "idle", "user_request"):
+            self._retry_state.pop(provider_id, None)
+
+        return []
+
+    def _handle_health_failed(self, event: HealthCheckFailed) -> List[Command]:
+        """
+        Handle health check failed event.
+
+        May trigger preemptive recovery for severely degraded providers.
+        """
+        # If failures are severe but provider not yet degraded, no action.
+        # The ProviderDegraded event will handle actual recovery.
+        return []
+
+    def _calculate_backoff(self, retry_count: int) -> float:
+        """Calculate backoff duration for a retry count."""
+        backoff = self._initial_backoff_s * (self._backoff_multiplier ** (retry_count - 1))
+        return min(backoff, self._max_backoff_s)
+
+    def get_retry_state(self, provider_id: str) -> Optional[Dict]:
+        """Get retry state for a provider (for monitoring)."""
+        return self._retry_state.get(provider_id)
+
+    def get_all_retry_states(self) -> Dict[str, Dict]:
+        """Get all retry states (for monitoring)."""
+        return dict(self._retry_state)
+
+    def reset_retry_state(self, provider_id: str) -> None:
+        """Manually reset retry state for a provider."""
+        self._retry_state.pop(provider_id, None)
+
+    def reset_all_retry_states(self) -> None:
+        """Reset all retry states."""
+        self._retry_state.clear()

mcp_hangar/application/services/__init__.py

@@ -0,0 +1,9 @@
+"""Application services - use case orchestration."""
+
+from .provider_service import ProviderService
+from .traced_provider_service import TracedProviderService
+
+__all__ = [
+    "ProviderService",
+    "TracedProviderService",
+]

mcp_hangar/application/services/provider_service.py

@@ -0,0 +1,208 @@
+"""Provider application service - orchestrates use cases."""
+
+from typing import Any, Dict, List
+
+from ...domain.exceptions import ProviderNotFoundError
+from ...domain.model import Provider
+from ...domain.repository import IProviderRepository
+from ...infrastructure.event_bus import EventBus
+from ...logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class ProviderService:
+    """
+    Application service for provider operations.
+
+    Orchestrates use cases by:
+    - Loading providers from repository
+    - Executing domain operations
+    - Publishing collected domain events
+    - Returning results
+    """
+
+    def __init__(
+        self,
+        repository: IProviderRepository,
+        event_bus: EventBus,
+    ):
+        self._repository = repository
+        self._event_bus = event_bus
+
+    def _publish_events(self, provider: Provider) -> None:
+        """Publish all collected events from provider."""
+        events = provider.collect_events()
+        for event in events:
+            try:
+                self._event_bus.publish(event)
+            except Exception as e:
+                logger.error(f"Failed to publish event {event.__class__.__name__}: {e}")
+
+    def _get_provider(self, provider_id: str) -> Provider:
+        """Get provider or raise ProviderNotFoundError."""
+        provider = self._repository.get(provider_id)
+        if provider is None:
+            raise ProviderNotFoundError(provider_id)
+        return provider
+
+    # --- Use Cases ---
+
+    def list_providers(self) -> List[Dict[str, Any]]:
+        """
+        Use case: List all providers with their status.
+
+        Returns:
+            List of provider status dictionaries
+        """
+        result = []
+        for provider_id, provider in self._repository.get_all().items():
+            result.append(provider.to_status_dict())
+        return result
+
+    def start_provider(self, provider_id: str) -> Dict[str, Any]:
+        """
+        Use case: Explicitly start a provider.
+
+        Ensures provider is ready and returns its status.
+
+        Args:
+            provider_id: Provider identifier
+
+        Returns:
+            Dictionary with provider state and tools
+
+        Raises:
+            ProviderNotFoundError: If provider doesn't exist
+        """
+        provider = self._get_provider(provider_id)
+        provider.ensure_ready()
+        self._publish_events(provider)
+
+        return {
+            "provider": provider_id,
+            "state": provider.state.value,
+            "tools": provider.get_tool_names(),
+        }
+
+    def stop_provider(self, provider_id: str) -> Dict[str, Any]:
+        """
+        Use case: Explicitly stop a provider.
+
+        Args:
+            provider_id: Provider identifier
+
+        Returns:
+            Confirmation dictionary
+
+        Raises:
+            ProviderNotFoundError: If provider doesn't exist
+        """
+        provider = self._get_provider(provider_id)
+        provider.shutdown()
+        self._publish_events(provider)
+
+        return {"stopped": provider_id}
+
+    def get_provider_tools(self, provider_id: str) -> Dict[str, Any]:
+        """
+        Use case: Get detailed tool schemas for a provider.
+
+        Ensures provider is ready before returning tools.
+
+        Args:
+            provider_id: Provider identifier
+
+        Returns:
+            Dictionary with provider ID and tool schemas
+
+        Raises:
+            ProviderNotFoundError: If provider doesn't exist
+        """
+        provider = self._get_provider(provider_id)
+        provider.ensure_ready()
+        self._publish_events(provider)
+
+        tools_list = []
+        for tool in provider.tools:
+            tools_list.append(tool.to_dict())
+
+        return {"provider": provider_id, "tools": tools_list}
+
+    def invoke_tool(
+        self,
+        provider_id: str,
+        tool_name: str,
+        arguments: Dict[str, Any],
+        timeout: float = 30.0,
+    ) -> Dict[str, Any]:
+        """
+        Use case: Invoke a tool on a provider.
+
+        Args:
+            provider_id: Provider identifier
+            tool_name: Tool name
+            arguments: Tool arguments
+            timeout: Timeout in seconds
+
+        Returns:
+            Tool result dictionary
+
+        Raises:
+            ProviderNotFoundError: If provider doesn't exist
+            ToolNotFoundError: If tool doesn't exist
+            ToolInvocationError: If invocation fails
+        """
+        provider = self._get_provider(provider_id)
+        result = provider.invoke_tool(tool_name, arguments, timeout)
+        self._publish_events(provider)
+
+        return result
+
+    def health_check(self, provider_id: str) -> bool:
+        """
+        Use case: Perform health check on a provider.
+
+        Args:
+            provider_id: Provider identifier
+
+        Returns:
+            True if healthy, False otherwise
+
+        Raises:
+            ProviderNotFoundError: If provider doesn't exist
+        """
+        provider = self._get_provider(provider_id)
+        healthy = provider.health_check()
+        self._publish_events(provider)
+
+        return healthy
+
+    def check_all_health(self) -> Dict[str, bool]:
+        """
+        Use case: Check health of all providers.
+
+        Returns:
+            Dictionary mapping provider_id to health status
+        """
+        results = {}
+        for provider_id, provider in self._repository.get_all().items():
+            results[provider_id] = provider.health_check()
+            self._publish_events(provider)
+
+        return results
+
+    def shutdown_idle_providers(self) -> List[str]:
+        """
+        Use case: Shutdown all idle providers.
+
+        Returns:
+            List of provider IDs that were shutdown
+        """
+        shutdown_ids = []
+        for provider_id, provider in self._repository.get_all().items():
+            if provider.maybe_shutdown_idle():
+                shutdown_ids.append(provider_id)
+                self._publish_events(provider)
+
+        return shutdown_ids