dory-sdk 2.1.0__py3-none-any.whl → 2.1.4__py3-none-any.whl

dory/edge/adaptive.py

@@ -0,0 +1,648 @@
+"""Adaptive behavior for edge vs cloud environments.
+
+Provides location-aware processing that automatically adjusts behavior
+based on whether the workload is running on edge or cloud nodes.
+"""
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Coroutine
+
+from dory.edge.detector import (
+    WorkloadContext,
+    WorkloadDetector,
+    NodeType,
+    get_workload_context,
+)
+from dory.edge.heartbeat import (
+    HeartbeatConfig,
+    HeartbeatManager,
+    ConnectivityStatus,
+)
+from dory.edge.fencing import FencingConfig, FencingManager
+from dory.edge.role import RoleManager, ProcessorRole
+
+logger = logging.getLogger(__name__)
+
+
+class OperationMode(Enum):
+    """Operating mode based on location and connectivity."""
+
+    EDGE_CONNECTED = "edge_connected"      # Edge node, orchestrator reachable
+    EDGE_OFFLINE = "edge_offline"          # Edge node, orchestrator unreachable
+    CLOUD_NORMAL = "cloud_normal"          # Cloud node, normal operation
+    CLOUD_FAILOVER = "cloud_failover"      # Cloud node, handling edge failover
+    DEGRADED = "degraded"                  # Unknown state, conservative mode
+
+
+@dataclass
+class EdgeConfig:
+    """Configuration for edge-specific behavior."""
+
+    # Heartbeat settings for edge (more aggressive)
+    edge_heartbeat_interval_sec: float = 5.0
+    edge_heartbeat_timeout_sec: float = 10.0
+    edge_max_missed_heartbeats: int = 3
+
+    # Heartbeat settings for cloud (standard)
+    cloud_heartbeat_interval_sec: float = 15.0
+    cloud_heartbeat_timeout_sec: float = 30.0
+    cloud_max_missed_heartbeats: int = 2
+
+    # Offline buffer settings (edge only)
+    offline_buffer_enabled: bool = True
+    offline_buffer_max_size_mb: int = 100
+    offline_buffer_flush_interval_sec: float = 30.0
+
+    # State checkpoint settings
+    edge_checkpoint_interval_sec: float = 10.0   # More frequent on edge
+    cloud_checkpoint_interval_sec: float = 60.0  # Less frequent on cloud
+
+    # Retry settings
+    edge_max_retries: int = 5
+    edge_retry_backoff_sec: float = 1.0
+    cloud_max_retries: int = 3
+    cloud_retry_backoff_sec: float = 2.0
+
+    # Resource constraints
+    edge_max_batch_size: int = 100      # Smaller batches on edge
+    cloud_max_batch_size: int = 1000    # Larger batches on cloud
+    edge_max_concurrent: int = 2        # Limited concurrency on edge
+    cloud_max_concurrent: int = 10      # Higher concurrency on cloud
+
+
+@dataclass
+class AdaptiveConfig:
+    """Full configuration for adaptive processor."""
+
+    app_name: str
+    processor_id: str
+    orchestrator_url: str | None = None
+    edge_config: EdgeConfig = field(default_factory=EdgeConfig)
+    fencing_config: FencingConfig | None = None
+    custom_detector: WorkloadDetector | None = None
+
+
+class AdaptiveProcessor:
+    """Processor that adapts behavior based on edge/cloud location.
+
+    Automatically adjusts:
+    - Heartbeat intervals and retry behavior
+    - State checkpoint frequency
+    - Batch sizes and concurrency
+    - Offline buffering (edge only)
+    - Failover handling
+
+    Usage:
+        processor = AdaptiveProcessor(AdaptiveConfig(
+            app_name="my-app",
+            processor_id="processor-1",
+            orchestrator_url="http://orchestrator:8080",
+        ))
+
+        await processor.start()
+
+        # Processor automatically adapts based on location
+        if processor.is_edge:
+            # Running on edge with edge-optimized settings
+            pass
+
+        if processor.is_offline:
+            # Edge node offline, using local buffer
+            pass
+    """
+
+    def __init__(self, config: AdaptiveConfig):
+        self.config = config
+        self._detector = config.custom_detector or WorkloadDetector()
+        self._context: WorkloadContext | None = None
+        self._mode = OperationMode.DEGRADED
+        self._heartbeat: HeartbeatManager | None = None
+        self._fencing: FencingManager | None = None
+        self._role_manager: RoleManager | None = None
+        self._offline_buffer: list[dict[str, Any]] = []
+        self._state: dict[str, Any] = {}
+        self._running = False
+        self._checkpoint_task: asyncio.Task | None = None
+        self._flush_task: asyncio.Task | None = None
+
+        # Callbacks
+        self._on_mode_change: Callable[[OperationMode, OperationMode], Coroutine] | None = None
+        self._on_connectivity_change: Callable[[ConnectivityStatus], Coroutine] | None = None
+        self._on_failover: Callable[[str], Coroutine] | None = None  # original_node
+        self._on_failback: Callable[[], Coroutine] | None = None
+
+    # =========================================================================
+    # Properties
+    # =========================================================================
+
+    @property
+    def context(self) -> WorkloadContext | None:
+        """Get current workload context."""
+        return self._context
+
+    @property
+    def mode(self) -> OperationMode:
+        """Get current operation mode."""
+        return self._mode
+
+    @property
+    def is_edge(self) -> bool:
+        """Check if running on edge node."""
+        return self._context.is_edge if self._context else False
+
+    @property
+    def is_cloud(self) -> bool:
+        """Check if running on cloud node."""
+        return not self.is_edge
+
+    @property
+    def is_migrated(self) -> bool:
+        """Check if this is a migrated (failover) workload."""
+        return self._context.is_migrated if self._context else False
+
+    @property
+    def is_offline(self) -> bool:
+        """Check if currently offline (edge only)."""
+        return self._mode == OperationMode.EDGE_OFFLINE
+
+    @property
+    def is_connected(self) -> bool:
+        """Check if connected to orchestrator."""
+        if self._heartbeat:
+            return self._heartbeat.is_connected()
+        return False
+
+    @property
+    def connectivity_status(self) -> ConnectivityStatus:
+        """Get current connectivity status."""
+        if self._heartbeat:
+            return self._heartbeat.get_status()
+        return ConnectivityStatus.UNKNOWN
+
+    @property
+    def role(self) -> ProcessorRole:
+        """Get current processor role."""
+        if self._role_manager:
+            return self._role_manager.role
+        return ProcessorRole.INITIALIZING
+
+    # =========================================================================
+    # Configuration Helpers
+    # =========================================================================
+
+    def get_heartbeat_config(self) -> HeartbeatConfig:
+        """Get heartbeat config based on location."""
+        ec = self.config.edge_config
+
+        if self.is_edge:
+            return HeartbeatConfig(
+                interval_sec=ec.edge_heartbeat_interval_sec,
+                timeout_sec=ec.edge_heartbeat_timeout_sec,
+                missed_threshold=ec.edge_max_missed_heartbeats,
+            )
+        else:
+            return HeartbeatConfig(
+                interval_sec=ec.cloud_heartbeat_interval_sec,
+                timeout_sec=ec.cloud_heartbeat_timeout_sec,
+                missed_threshold=ec.cloud_max_missed_heartbeats,
+            )
+
+    def get_checkpoint_interval(self) -> float:
+        """Get checkpoint interval based on location."""
+        ec = self.config.edge_config
+        return ec.edge_checkpoint_interval_sec if self.is_edge else ec.cloud_checkpoint_interval_sec
+
+    def get_max_batch_size(self) -> int:
+        """Get max batch size based on location."""
+        ec = self.config.edge_config
+        return ec.edge_max_batch_size if self.is_edge else ec.cloud_max_batch_size
+
+    def get_max_concurrent(self) -> int:
+        """Get max concurrency based on location."""
+        ec = self.config.edge_config
+        return ec.edge_max_concurrent if self.is_edge else ec.cloud_max_concurrent
+
+    def get_retry_config(self) -> tuple[int, float]:
+        """Get retry config (max_retries, backoff_sec) based on location."""
+        ec = self.config.edge_config
+        if self.is_edge:
+            return ec.edge_max_retries, ec.edge_retry_backoff_sec
+        return ec.cloud_max_retries, ec.cloud_retry_backoff_sec
+
+    # =========================================================================
+    # Lifecycle
+    # =========================================================================
+
+    async def start(self) -> OperationMode:
+        """Start the adaptive processor.
+
+        Returns:
+            Initial operation mode
+        """
+        logger.info(f"Starting adaptive processor: {self.config.processor_id}")
+
+        # Detect workload context
+        self._context = self._detector.detect()
+        logger.info(
+            f"Workload context: node_type={self._context.node_type.value}, "
+            f"is_edge={self._context.is_edge}, is_migrated={self._context.is_migrated}"
+        )
+
+        # Initialize fencing
+        fencing_config = self.config.fencing_config or FencingConfig()
+        self._fencing = FencingManager(
+            app_name=self.config.app_name,
+            config=fencing_config,
+        )
+
+        # Initialize role manager
+        self._role_manager = RoleManager(
+            processor_id=self.config.processor_id,
+            fencing_manager=self._fencing,
+        )
+
+        # Initialize heartbeat if orchestrator URL provided
+        if self.config.orchestrator_url:
+            heartbeat_config = self.get_heartbeat_config()
+            self._heartbeat = HeartbeatManager(
+                processor_id=self.config.processor_id,
+                orchestrator_url=self.config.orchestrator_url,
+                config=heartbeat_config,
+            )
+
+            # Register connectivity callback
+            if self._on_connectivity_change:
+                # HeartbeatManager would need to support this
+                pass
+
+        # Handle migrated workload
+        if self._context.is_migrated:
+            logger.info(f"Migrated workload from: {self._context.original_node}")
+            if self._on_failover:
+                await self._on_failover(self._context.original_node or "unknown")
+
+        # Start role manager (acquires fencing token)
+        await self._role_manager.start()
+
+        # Start heartbeat
+        if self._heartbeat:
+            await self._heartbeat.start()
+
+        # Determine initial mode
+        self._mode = self._determine_mode()
+        logger.info(f"Initial operation mode: {self._mode.value}")
+
+        # Start background tasks
+        self._running = True
+        self._checkpoint_task = asyncio.create_task(self._checkpoint_loop())
+
+        if self.is_edge and self.config.edge_config.offline_buffer_enabled:
+            self._flush_task = asyncio.create_task(self._flush_loop())
+
+        return self._mode
+
+    async def stop(self, reason: str = "shutdown") -> None:
+        """Stop the adaptive processor.
+
+        Args:
+            reason: Reason for stopping
+        """
+        logger.info(f"Stopping adaptive processor: {reason}")
+        self._running = False
+
+        # Cancel background tasks
+        if self._checkpoint_task:
+            self._checkpoint_task.cancel()
+            try:
+                await self._checkpoint_task
+            except asyncio.CancelledError:
+                pass
+
+        if self._flush_task:
+            self._flush_task.cancel()
+            try:
+                await self._flush_task
+            except asyncio.CancelledError:
+                pass
+
+        # Flush any remaining offline buffer
+        if self._offline_buffer:
+            await self._flush_offline_buffer()
+
+        # Final state checkpoint
+        await self._save_state()
+
+        # Stop heartbeat
+        if self._heartbeat:
+            await self._heartbeat.stop()
+
+        # Release fencing
+        if self._role_manager:
+            await self._role_manager.stop(reason)
+
+        logger.info("Adaptive processor stopped")
+
+    # =========================================================================
+    # Mode Management
+    # =========================================================================
+
+    def _determine_mode(self) -> OperationMode:
+        """Determine current operation mode."""
+        if not self._context:
+            return OperationMode.DEGRADED
+
+        if self._context.is_edge:
+            # Edge node
+            if self.is_connected:
+                return OperationMode.EDGE_CONNECTED
+            else:
+                return OperationMode.EDGE_OFFLINE
+        else:
+            # Cloud node
+            if self._context.is_migrated:
+                return OperationMode.CLOUD_FAILOVER
+            else:
+                return OperationMode.CLOUD_NORMAL
+
+    async def _update_mode(self) -> None:
+        """Update operation mode and trigger callbacks if changed."""
+        old_mode = self._mode
+        new_mode = self._determine_mode()
+
+        if old_mode != new_mode:
+            logger.info(f"Mode change: {old_mode.value} -> {new_mode.value}")
+            self._mode = new_mode
+
+            if self._on_mode_change:
+                await self._on_mode_change(old_mode, new_mode)
+
+            # Handle specific transitions
+            if old_mode == OperationMode.EDGE_OFFLINE and new_mode == OperationMode.EDGE_CONNECTED:
+                # Came back online - flush buffer
+                await self._flush_offline_buffer()
+
+    # =========================================================================
+    # State Management
+    # =========================================================================
+
+    def get_state(self) -> dict[str, Any]:
+        """Get current processor state."""
+        return self._state.copy()
+
+    def set_state(self, state: dict[str, Any]) -> None:
+        """Set processor state."""
+        self._state = state.copy()
+
+    async def restore_state(self, state: dict[str, Any]) -> None:
+        """Restore state (called during failover recovery)."""
+        self._state = state.copy()
+        logger.info(f"State restored: {len(state)} keys")
+
+    async def _save_state(self) -> None:
+        """Save state to appropriate storage."""
+        if self.is_offline:
+            # Buffer locally when offline
+            self._buffer_state(self._state)
+        else:
+            # Save to remote storage
+            await self._save_state_remote(self._state)
+
+    async def _save_state_remote(self, state: dict[str, Any]) -> None:
+        """Save state to remote storage (orchestrator/S3)."""
+        # This would integrate with StateManager
+        logger.debug(f"Saving state remotely: {len(state)} keys")
+
+    def _buffer_state(self, state: dict[str, Any]) -> None:
+        """Buffer state locally (edge offline mode)."""
+        max_size = self.config.edge_config.offline_buffer_max_size_mb * 1024 * 1024
+        # Simplified size check
+        if len(self._offline_buffer) < 10000:  # Rough limit
+            self._offline_buffer.append(state.copy())
+            logger.debug(f"State buffered locally: {len(self._offline_buffer)} items")
+
+    async def _flush_offline_buffer(self) -> None:
+        """Flush offline buffer to remote storage."""
+        if not self._offline_buffer:
+            return
+
+        logger.info(f"Flushing offline buffer: {len(self._offline_buffer)} items")
+
+        # In real implementation, this would batch-upload to S3/orchestrator
+        flushed = 0
+        while self._offline_buffer and self.is_connected:
+            state = self._offline_buffer.pop(0)
+            try:
+                await self._save_state_remote(state)
+                flushed += 1
+            except Exception as e:
+                # Put back and retry later
+                self._offline_buffer.insert(0, state)
+                logger.warning(f"Buffer flush failed: {e}")
+                break
+
+        logger.info(f"Flushed {flushed} items from offline buffer")
+
+    # =========================================================================
+    # Background Tasks
+    # =========================================================================
+
+    async def _checkpoint_loop(self) -> None:
+        """Periodic state checkpoint."""
+        interval = self.get_checkpoint_interval()
+
+        while self._running:
+            try:
+                await asyncio.sleep(interval)
+                await self._save_state()
+                await self._update_mode()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Checkpoint error: {e}")
+
+    async def _flush_loop(self) -> None:
+        """Periodic offline buffer flush (edge only)."""
+        interval = self.config.edge_config.offline_buffer_flush_interval_sec
+
+        while self._running:
+            try:
+                await asyncio.sleep(interval)
+                if self.is_connected and self._offline_buffer:
+                    await self._flush_offline_buffer()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Flush error: {e}")
+
+    # =========================================================================
+    # Callbacks
+    # =========================================================================
+
+    def on_mode_change(
+        self, callback: Callable[[OperationMode, OperationMode], Coroutine]
+    ) -> None:
+        """Register callback for mode changes."""
+        self._on_mode_change = callback
+
+    def on_connectivity_change(
+        self, callback: Callable[[ConnectivityStatus], Coroutine]
+    ) -> None:
+        """Register callback for connectivity changes."""
+        self._on_connectivity_change = callback
+
+    def on_failover(self, callback: Callable[[str], Coroutine]) -> None:
+        """Register callback for failover (receives original_node)."""
+        self._on_failover = callback
+
+    def on_failback(self, callback: Callable[[], Coroutine]) -> None:
+        """Register callback for failback to edge."""
+        self._on_failback = callback
+
+    # =========================================================================
+    # Processing Helpers
+    # =========================================================================
+
+    async def process_with_fencing(
+        self,
+        operation: Callable[[], Coroutine[Any, Any, Any]],
+    ) -> Any:
+        """Execute operation with fencing validation.
+
+        Validates fencing token before and after operation to ensure
+        we're still the primary processor.
+
+        Args:
+            operation: Async operation to execute
+
+        Returns:
+            Operation result
+
+        Raises:
+            FenceViolation: If fencing is violated
+        """
+        if not self._role_manager:
+            raise RuntimeError("Processor not started")
+
+        # Validate before
+        if not await self._role_manager.validate_fencing_or_fence():
+            from dory.edge.fencing import FenceViolation
+            raise FenceViolation("Fencing validation failed before operation")
+
+        # Execute operation
+        result = await operation()
+
+        # Validate after
+        if not await self._role_manager.validate_fencing_or_fence():
+            from dory.edge.fencing import FenceViolation
+            raise FenceViolation("Fencing validation failed after operation")
+
+        return result
+
+    async def process_batch(
+        self,
+        items: list[Any],
+        processor: Callable[[Any], Coroutine[Any, Any, Any]],
+    ) -> list[Any]:
+        """Process items in batches appropriate for current location.
+
+        Automatically adjusts batch size and concurrency based on
+        whether running on edge or cloud.
+
+        Args:
+            items: Items to process
+            processor: Async function to process each item
+
+        Returns:
+            List of results
+        """
+        batch_size = self.get_max_batch_size()
+        max_concurrent = self.get_max_concurrent()
+
+        results = []
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def process_with_semaphore(item: Any) -> Any:
+            async with semaphore:
+                return await processor(item)
+
+        # Process in batches
+        for i in range(0, len(items), batch_size):
+            batch = items[i:i + batch_size]
+            batch_results = await asyncio.gather(
+                *[process_with_semaphore(item) for item in batch],
+                return_exceptions=True,
+            )
+            results.extend(batch_results)
+
+            # Checkpoint after each batch
+            if self.is_edge:
+                await self._save_state()
+
+        return results
+
+
+# =============================================================================
+# Convenience Functions
+# =============================================================================
+
+def create_adaptive_processor(
+    app_name: str,
+    processor_id: str,
+    orchestrator_url: str | None = None,
+    **kwargs: Any,
+) -> AdaptiveProcessor:
+    """Create an adaptive processor with sensible defaults.
+
+    Args:
+        app_name: Application name
+        processor_id: Unique processor identifier
+        orchestrator_url: URL of the orchestrator service
+        **kwargs: Additional EdgeConfig parameters
+
+    Returns:
+        Configured AdaptiveProcessor
+    """
+    edge_config = EdgeConfig(**kwargs) if kwargs else EdgeConfig()
+
+    config = AdaptiveConfig(
+        app_name=app_name,
+        processor_id=processor_id,
+        orchestrator_url=orchestrator_url,
+        edge_config=edge_config,
+    )
+
+    return AdaptiveProcessor(config)
+
+
+def get_location_aware_settings() -> dict[str, Any]:
+    """Get settings appropriate for current location.
+
+    Returns:
+        Dictionary of settings adjusted for edge or cloud
+    """
+    context = get_workload_context()
+    config = EdgeConfig()
+
+    if context.is_edge:
+        return {
+            "heartbeat_interval_sec": config.edge_heartbeat_interval_sec,
+            "checkpoint_interval_sec": config.edge_checkpoint_interval_sec,
+            "max_batch_size": config.edge_max_batch_size,
+            "max_concurrent": config.edge_max_concurrent,
+            "max_retries": config.edge_max_retries,
+            "retry_backoff_sec": config.edge_retry_backoff_sec,
+            "offline_buffer_enabled": config.offline_buffer_enabled,
+        }
+    else:
+        return {
+            "heartbeat_interval_sec": config.cloud_heartbeat_interval_sec,
+            "checkpoint_interval_sec": config.cloud_checkpoint_interval_sec,
+            "max_batch_size": config.cloud_max_batch_size,
+            "max_concurrent": config.cloud_max_concurrent,
+            "max_retries": config.cloud_max_retries,
+            "retry_backoff_sec": config.cloud_retry_backoff_sec,
+            "offline_buffer_enabled": False,
+        }