dory-sdk 2.1.0__py3-none-any.whl → 2.1.4__py3-none-any.whl

dory/edge/role.py

@@ -0,0 +1,416 @@
+"""Role management for edge processor failover.
+
+Provides role-based coordination between edge and cloud instances
+during failover scenarios.
+"""
+
+import asyncio
+import logging
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Awaitable
+
+from dory.edge.fencing import (
+    FencingManager,
+    FencingToken,
+    FencingConfig,
+    FenceViolation,
+    StaleEpochError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessorRole(Enum):
+    """Role states for a processor instance.
+
+    Transitions:
+        INITIALIZING -> PRIMARY (normal startup)
+        INITIALIZING -> STANDBY (cloud replica waiting for failover)
+        PRIMARY -> DRAINING (graceful shutdown or failover)
+        PRIMARY -> FENCED (lost fencing token)
+        DRAINING -> STOPPED
+        STANDBY -> PRIMARY (failover promoted)
+        FENCED -> STOPPED
+    """
+
+    INITIALIZING = "initializing"
+    PRIMARY = "primary"
+    STANDBY = "standby"
+    DRAINING = "draining"
+    FENCED = "fenced"
+    STOPPED = "stopped"
+
+    def can_process(self) -> bool:
+        """Check if this role allows message processing."""
+        return self == ProcessorRole.PRIMARY
+
+    def can_write_state(self) -> bool:
+        """Check if this role allows state modifications."""
+        return self in (ProcessorRole.PRIMARY, ProcessorRole.DRAINING)
+
+    def is_terminal(self) -> bool:
+        """Check if this is a terminal (final) state."""
+        return self in (ProcessorRole.STOPPED, ProcessorRole.FENCED)
+
+
+@dataclass
+class RoleTransition:
+    """Record of a role transition event."""
+
+    from_role: ProcessorRole
+    to_role: ProcessorRole
+    timestamp: float = field(default_factory=time.time)
+    reason: str = ""
+    epoch: int | None = None
+    node_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to dictionary."""
+        return {
+            "from_role": self.from_role.value,
+            "to_role": self.to_role.value,
+            "timestamp": self.timestamp,
+            "reason": self.reason,
+            "epoch": self.epoch,
+            "node_id": self.node_id,
+        }
+
+
+# Type alias for role change callbacks
+RoleChangeCallback = Callable[[RoleTransition], Awaitable[None]]
+
+
+class RoleManager:
+    """Manager for processor role transitions and fencing coordination.
+
+    Coordinates role-based behavior for edge-to-cloud failover:
+    - Edge processor starts as PRIMARY with fencing token
+    - Cloud replica starts as STANDBY
+    - On edge failure, cloud acquires fencing token and becomes PRIMARY
+    - Edge instance becomes FENCED when it loses the token
+
+    Usage:
+        fencing_config = FencingConfig(backend="redis")
+        manager = RoleManager(
+            processor_id="my-processor",
+            node_id="edge-node-1",
+            fencing_config=fencing_config,
+        )
+
+        # Start and acquire PRIMARY role
+        await manager.start()
+        assert manager.role == ProcessorRole.PRIMARY
+
+        # Check role before processing
+        if manager.can_process():
+            process_message(msg)
+
+        # Handle failover (e.g., when losing connectivity)
+        manager.add_role_change_callback(handle_role_change)
+
+        # Graceful shutdown
+        await manager.stop()
+    """
+
+    def __init__(
+        self,
+        processor_id: str,
+        node_id: str,
+        fencing_config: FencingConfig | None = None,
+        start_as_standby: bool = False,
+    ):
+        """Initialize role manager.
+
+        Args:
+            processor_id: Unique processor identifier
+            node_id: Current node identifier (e.g., pod name)
+            fencing_config: Fencing configuration
+            start_as_standby: If True, start in STANDBY role
+        """
+        self._processor_id = processor_id
+        self._node_id = node_id
+        self._start_as_standby = start_as_standby
+
+        self._fencing_manager = FencingManager(fencing_config)
+        self._fencing_token: FencingToken | None = None
+
+        self._role = ProcessorRole.INITIALIZING
+        self._role_history: list[RoleTransition] = []
+        self._callbacks: list[RoleChangeCallback] = []
+
+        self._monitor_task: asyncio.Task | None = None
+        self._running = False
+
+    @property
+    def role(self) -> ProcessorRole:
+        """Current processor role."""
+        return self._role
+
+    @property
+    def processor_id(self) -> str:
+        """Processor identifier."""
+        return self._processor_id
+
+    @property
+    def node_id(self) -> str:
+        """Node identifier."""
+        return self._node_id
+
+    @property
+    def fencing_token(self) -> FencingToken | None:
+        """Current fencing token if PRIMARY."""
+        return self._fencing_token
+
+    def can_process(self) -> bool:
+        """Check if processor can handle messages."""
+        return self._role.can_process()
+
+    def can_write_state(self) -> bool:
+        """Check if processor can modify state."""
+        return self._role.can_write_state()
+
+    def add_role_change_callback(self, callback: RoleChangeCallback) -> None:
+        """Register callback for role changes.
+
+        Args:
+            callback: Async function called on role transitions
+        """
+        self._callbacks.append(callback)
+
+    def remove_role_change_callback(self, callback: RoleChangeCallback) -> None:
+        """Remove role change callback."""
+        if callback in self._callbacks:
+            self._callbacks.remove(callback)
+
+    async def start(self) -> ProcessorRole:
+        """Start role management and acquire initial role.
+
+        Returns:
+            Initial role (PRIMARY or STANDBY)
+        """
+        self._running = True
+
+        if self._start_as_standby:
+            await self._transition_to(
+                ProcessorRole.STANDBY,
+                reason="Started as standby replica",
+            )
+        else:
+            # Try to acquire PRIMARY role with fencing token
+            try:
+                self._fencing_token = await self._fencing_manager.acquire(
+                    self._processor_id,
+                    self._node_id,
+                )
+                await self._transition_to(
+                    ProcessorRole.PRIMARY,
+                    reason="Acquired fencing token",
+                    epoch=self._fencing_token.epoch,
+                )
+            except Exception as e:
+                logger.error(f"Failed to acquire fencing token: {e}")
+                await self._transition_to(
+                    ProcessorRole.STANDBY,
+                    reason=f"Failed to acquire fencing token: {e}",
+                )
+
+        # Start background monitoring
+        self._monitor_task = asyncio.create_task(self._monitor_fencing())
+
+        return self._role
+
+    async def stop(self, reason: str = "Shutdown requested") -> None:
+        """Stop role management and release resources.
+
+        Args:
+            reason: Reason for stopping
+        """
+        self._running = False
+
+        # Stop monitoring
+        if self._monitor_task:
+            self._monitor_task.cancel()
+            try:
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            self._monitor_task = None
+
+        # Drain if PRIMARY
+        if self._role == ProcessorRole.PRIMARY:
+            await self._transition_to(
+                ProcessorRole.DRAINING,
+                reason=reason,
+            )
+
+        # Release fencing token
+        if self._fencing_token:
+            await self._fencing_manager.release(self._fencing_token)
+            self._fencing_token = None
+
+        await self._transition_to(
+            ProcessorRole.STOPPED,
+            reason=reason,
+        )
+
+        # Close fencing manager
+        await self._fencing_manager.close()
+
+    async def promote_to_primary(self) -> bool:
+        """Attempt to promote from STANDBY to PRIMARY.
+
+        Returns:
+            True if promotion succeeded
+        """
+        if self._role != ProcessorRole.STANDBY:
+            logger.warning(
+                f"Cannot promote from {self._role.value}, "
+                "must be STANDBY"
+            )
+            return False
+
+        try:
+            # Acquire fencing token
+            self._fencing_token = await self._fencing_manager.acquire(
+                self._processor_id,
+                self._node_id,
+            )
+
+            await self._transition_to(
+                ProcessorRole.PRIMARY,
+                reason="Promoted from standby",
+                epoch=self._fencing_token.epoch,
+            )
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Failed to promote to PRIMARY: {e}")
+            return False
+
+    async def demote_to_standby(self, reason: str = "Demoted") -> None:
+        """Demote from PRIMARY to STANDBY, releasing fencing token.
+
+        Args:
+            reason: Reason for demotion
+        """
+        if self._role not in (ProcessorRole.PRIMARY, ProcessorRole.DRAINING):
+            logger.warning(f"Cannot demote from {self._role.value}")
+            return
+
+        # Release fencing token
+        if self._fencing_token:
+            await self._fencing_manager.release(self._fencing_token)
+            self._fencing_token = None
+
+        await self._transition_to(
+            ProcessorRole.STANDBY,
+            reason=reason,
+        )
+
+    async def validate_fencing(self) -> bool:
+        """Validate current fencing token is still valid.
+
+        Returns:
+            True if fencing token is valid (or not PRIMARY)
+        """
+        if self._role != ProcessorRole.PRIMARY:
+            return True
+
+        if not self._fencing_token:
+            return False
+
+        return await self._fencing_manager.validate(self._fencing_token)
+
+    async def validate_fencing_or_fence(self) -> bool:
+        """Validate fencing and transition to FENCED if stale.
+
+        Returns:
+            True if still PRIMARY, False if fenced
+        """
+        if not await self.validate_fencing():
+            await self._transition_to(
+                ProcessorRole.FENCED,
+                reason="Fencing token became stale",
+            )
+            return False
+        return True
+
+    async def _transition_to(
+        self,
+        new_role: ProcessorRole,
+        reason: str = "",
+        epoch: int | None = None,
+    ) -> None:
+        """Transition to a new role.
+
+        Args:
+            new_role: Target role
+            reason: Reason for transition
+            epoch: Fencing epoch if applicable
+        """
+        if new_role == self._role:
+            return
+
+        old_role = self._role
+        self._role = new_role
+
+        transition = RoleTransition(
+            from_role=old_role,
+            to_role=new_role,
+            reason=reason,
+            epoch=epoch or (self._fencing_token.epoch if self._fencing_token else None),
+            node_id=self._node_id,
+        )
+        self._role_history.append(transition)
+
+        logger.info(
+            f"Role transition: {old_role.value} -> {new_role.value} "
+            f"(reason={reason}, epoch={transition.epoch})"
+        )
+
+        # Notify callbacks
+        for callback in self._callbacks:
+            try:
+                await callback(transition)
+            except Exception as e:
+                logger.error(f"Role change callback failed: {e}")
+
+    async def _monitor_fencing(self) -> None:
+        """Background task to monitor fencing token validity."""
+        while self._running:
+            try:
+                await asyncio.sleep(5.0)  # Check every 5 seconds
+
+                if self._role == ProcessorRole.PRIMARY:
+                    is_valid = await self.validate_fencing()
+                    if not is_valid:
+                        logger.warning("Fencing token validation failed")
+                        await self._transition_to(
+                            ProcessorRole.FENCED,
+                            reason="Fencing token invalidated by another instance",
+                        )
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Fencing monitor error: {e}")
+
+    def get_role_history(self) -> list[RoleTransition]:
+        """Get list of role transitions."""
+        return self._role_history.copy()
+
+    def get_status(self) -> dict[str, Any]:
+        """Get current role status as dictionary."""
+        return {
+            "processor_id": self._processor_id,
+            "node_id": self._node_id,
+            "role": self._role.value,
+            "can_process": self.can_process(),
+            "can_write_state": self.can_write_state(),
+            "fencing_epoch": self._fencing_token.epoch if self._fencing_token else None,
+            "is_running": self._running,
+            "transition_count": len(self._role_history),
+        }