dory-processor-sdk 0.0.1__py3-none-any.whl

dory/simple.py

@@ -0,0 +1,342 @@
+"""
+Simple function-based API for Dory processors.
+
+For applications that don't need the full class-based API,
+this module provides a simpler decorator-based approach.
+
+Usage:
+    from dory.simple import processor, state
+
+    counter = state(0)
+
+    @processor
+    async def main(ctx):
+        async for _ in ctx.run_loop(interval=1):
+            counter.value += 1
+"""
+
+import asyncio
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, Callable, Awaitable, TypeVar, Generic
+
+from dory.core.app import DoryApp
+from dory.core.processor import BaseProcessor
+from dory.core.context import ExecutionContext
+
+T = TypeVar('T')
+
+
+@dataclass
+class StateVar(Generic[T]):
+    """
+    A state variable for function-based processors.
+
+    Usage:
+        counter = state(0)
+        sessions = state(dict)  # Factory for mutable defaults
+
+        @processor
+        async def main(ctx):
+            counter.value += 1
+            sessions.value["user1"] = data
+    """
+    _default: T | Callable[[], T]
+    _value: T = field(default=None, init=False, repr=False)
+    _initialized: bool = field(default=False, init=False)
+
+    @property
+    def value(self) -> T:
+        """Get the current value."""
+        if not self._initialized:
+            if callable(self._default):
+                self._value = self._default()
+            else:
+                self._value = self._default
+            self._initialized = True
+        return self._value
+
+    @value.setter
+    def value(self, new_value: T) -> None:
+        """Set a new value."""
+        self._value = new_value
+        self._initialized = True
+
+    def reset(self) -> None:
+        """Reset to default value."""
+        self._initialized = False
+
+    def get(self) -> T:
+        """Alias for value property."""
+        return self.value
+
+    def set(self, new_value: T) -> None:
+        """Alias for value setter."""
+        self.value = new_value
+
+
+def state(default: T | Callable[[], T] = None) -> StateVar[T]:
+    """
+    Create a state variable for function-based processors.
+
+    State variables are automatically saved and restored during migrations.
+
+    Args:
+        default: Default value or factory function for mutable defaults
+
+    Usage:
+        # Simple values
+        counter = state(0)
+        name = state("default")
+
+        # Mutable values (use factory)
+        data = state(dict)   # Creates new dict each time
+        items = state(list)  # Creates new list each time
+
+        @processor
+        async def main(ctx):
+            counter.value += 1
+            data.value["key"] = "value"
+    """
+    return StateVar(_default=default)
+
+
+# Global registry of state variables for current processor
+_state_registry: dict[str, StateVar] = {}
+
+
+def _register_state(name: str, var: StateVar) -> None:
+    """Register a state variable (called by processor decorator)."""
+    _state_registry[name] = var
+
+
+def _get_all_state() -> dict[str, Any]:
+    """Get all state values."""
+    return {name: var.value for name, var in _state_registry.items()}
+
+
+def _set_all_state(values: dict[str, Any]) -> None:
+    """Set all state values."""
+    for name, value in values.items():
+        if name in _state_registry:
+            _state_registry[name].value = value
+
+
+class FunctionProcessor(BaseProcessor):
+    """
+    Wrapper that converts a function into a BaseProcessor.
+
+    Internal use by @processor decorator.
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ExecutionContext], Awaitable[None]],
+        state_vars: dict[str, StateVar],
+        context: ExecutionContext | None = None,
+    ):
+        super().__init__(context)
+        self._func = func
+        self._state_vars = state_vars
+
+    async def run(self) -> None:
+        """Run the wrapped function."""
+        await self._func(self.context)
+
+    def get_state(self) -> dict:
+        """Get state from registered state variables."""
+        return {name: var.value for name, var in self._state_vars.items()}
+
+    async def restore_state(self, state: dict) -> None:
+        """Restore state to registered state variables."""
+        for name, value in state.items():
+            if name in self._state_vars:
+                self._state_vars[name].value = value
+
+
+class ContextWrapper:
+    """
+    Wrapper around ExecutionContext with additional helpers for function-based API.
+
+    Provides run_loop() and other conveniences directly on ctx.
+    """
+
+    def __init__(self, context: ExecutionContext):
+        self._context = context
+
+    def __getattr__(self, name: str) -> Any:
+        """Delegate to underlying context."""
+        return getattr(self._context, name)
+
+    async def run_loop(
+        self,
+        interval: float = 1.0,
+        check_migration: bool = True,
+    ):
+        """
+        Async iterator that yields until shutdown is requested.
+
+        Usage:
+            @processor
+            async def main(ctx):
+                async for i in ctx.run_loop(interval=1):
+                    counter.value += 1
+        """
+        iteration = 0
+        while not self._context.is_shutdown_requested():
+            yield iteration
+            iteration += 1
+
+            if check_migration and self._context.is_migration_imminent():
+                self._context.logger().info(
+                    f"Migration imminent, completing iteration {iteration}"
+                )
+
+            await asyncio.sleep(interval)
+
+    @property
+    def shutdown_requested(self) -> bool:
+        """Check if shutdown is requested (simpler than is_shutdown_requested())."""
+        return self._context.is_shutdown_requested()
+
+    @property
+    def migration_imminent(self) -> bool:
+        """Check if migration is imminent (simpler than is_migration_imminent())."""
+        return self._context.is_migration_imminent()
+
+
+def processor(
+    func: Callable[[ContextWrapper], Awaitable[None]] | None = None,
+    *,
+    config_file: str | None = None,
+    log_level: str | None = None,
+):
+    """
+    Decorator to create a Dory processor from a simple async function.
+
+    This is the simplest way to create a Dory processor. Just decorate
+    your main async function and it handles everything else.
+
+    Args:
+        func: The async function to wrap
+        config_file: Optional path to config file
+        log_level: Optional log level override
+
+    Usage:
+        # Minimal stateless processor
+        from dory.simple import processor
+
+        @processor
+        async def main(ctx):
+            while not ctx.shutdown_requested:
+                print("Working...")
+                await asyncio.sleep(1)
+
+        # With state
+        from dory.simple import processor, state
+
+        counter = state(0)
+        data = state(dict)
+
+        @processor
+        async def main(ctx):
+            async for i in ctx.run_loop(interval=1):
+                counter.value += 1
+                print(f"Count: {counter.value}")
+
+        # With config
+        @processor(config_file="dory.yaml", log_level="DEBUG")
+        async def main(ctx):
+            ...
+    """
+    def decorator(fn: Callable[[ContextWrapper], Awaitable[None]]):
+        # Collect state variables from the module
+        frame = inspect.currentframe()
+        if frame and frame.f_back and frame.f_back.f_back:
+            module_globals = frame.f_back.f_back.f_globals
+            state_vars = {
+                name: var
+                for name, var in module_globals.items()
+                if isinstance(var, StateVar)
+            }
+        else:
+            state_vars = {}
+
+        # Create wrapper function that wraps context
+        async def wrapped_func(context: ExecutionContext) -> None:
+            wrapper = ContextWrapper(context)
+            await fn(wrapper)
+
+        # Create processor class dynamically
+        class DynamicProcessor(FunctionProcessor):
+            def __init__(self, context: ExecutionContext | None = None):
+                super().__init__(wrapped_func, state_vars, context)
+
+        # Run immediately if this is the main module
+        if frame and frame.f_back and frame.f_back.f_back:
+            if module_globals.get("__name__") == "__main__":
+                DoryApp(
+                    config_file=config_file,
+                    log_level=log_level,
+                ).run(DynamicProcessor)
+
+        return fn
+
+    if func is not None:
+        # Called without arguments: @processor
+        return decorator(func)
+    else:
+        # Called with arguments: @processor(config_file="...")
+        return decorator
+
+
+def run_processor(
+    func: Callable[[ContextWrapper], Awaitable[None]],
+    *,
+    config_file: str | None = None,
+    log_level: str | None = None,
+) -> None:
+    """
+    Run a function as a Dory processor.
+
+    Alternative to @processor decorator when you want explicit control.
+
+    Usage:
+        from dory.simple import run_processor, state
+
+        counter = state(0)
+
+        async def main(ctx):
+            async for _ in ctx.run_loop(interval=1):
+                counter.value += 1
+
+        if __name__ == "__main__":
+            run_processor(main)
+    """
+    # Collect state variables from caller's module
+    frame = inspect.currentframe()
+    if frame and frame.f_back:
+        module_globals = frame.f_back.f_globals
+        state_vars = {
+            name: var
+            for name, var in module_globals.items()
+            if isinstance(var, StateVar)
+        }
+    else:
+        state_vars = {}
+
+    # Create wrapper function
+    async def wrapped_func(context: ExecutionContext) -> None:
+        wrapper = ContextWrapper(context)
+        await func(wrapper)
+
+    # Create processor class
+    class DynamicProcessor(FunctionProcessor):
+        def __init__(self, context: ExecutionContext | None = None):
+            super().__init__(wrapped_func, state_vars, context)
+
+    # Run
+    DoryApp(
+        config_file=config_file,
+        log_level=log_level,
+    ).run(DynamicProcessor)

dory/types.py

@@ -0,0 +1,68 @@
+"""
+Type definitions for Dory SDK.
+
+This module contains all type hints and type aliases used across the SDK.
+No application-specific types should be defined here.
+"""
+
+from typing import TypeVar, Callable, Awaitable, Any, Protocol
+from enum import Enum, auto
+
+
+class LifecycleState(Enum):
+    """States in the processor lifecycle state machine."""
+
+    CREATED = auto()       # Instance created, not yet started
+    STARTING = auto()      # startup() in progress
+    RUNNING = auto()       # run() in progress
+    SHUTTING_DOWN = auto() # shutdown() in progress
+    STOPPED = auto()       # Gracefully stopped
+    FAILED = auto()        # Error occurred
+
+
+class StateBackend(Enum):
+    """Supported state storage backends."""
+
+    CONFIGMAP = "configmap"  # Kubernetes ConfigMap (default, <1MB)
+    PVC = "pvc"              # Persistent Volume Claim
+    S3 = "s3"                # AWS S3 (for multi-cluster)
+    LOCAL = "local"          # Local file (for testing)
+
+
+class RecoveryStrategy(Enum):
+    """Recovery strategies after fault detection."""
+
+    RESTORE_STATE = "restore_state"          # Try to restore from checkpoint
+    GOLDEN_IMAGE = "golden_image"            # Start fresh, discard state
+    GOLDEN_WITH_BACKOFF = "golden_backoff"   # Start fresh with delay
+
+
+class FaultType(Enum):
+    """Types of faults detected by SDK."""
+
+    CLEAN_EXIT = "clean_exit"           # Normal termination
+    UNEXPECTED_CRASH = "crash"          # Exit code non-zero
+    HUNG_PROCESS = "hung"               # Liveness probe timeout
+    STATE_CORRUPTION = "state_corrupt"  # State validation failed
+    STARTUP_FAILURE = "startup_fail"    # Startup threw exception
+    HEALTH_CHECK_FAILURE = "health_fail"  # Health endpoint returned error
+
+
+# Type aliases
+StateDict = dict[str, Any]
+ConfigDict = dict[str, Any]
+MetadataDict = dict[str, str]
+
+# Callback types
+ShutdownCallback = Callable[[], Awaitable[None]]
+StateCallback = Callable[[], StateDict]
+
+
+class ProcessorProtocol(Protocol):
+    """Protocol defining the processor interface."""
+
+    async def startup(self) -> None: ...
+    async def run(self) -> None: ...
+    async def shutdown(self) -> None: ...
+    def get_state(self) -> StateDict: ...
+    async def restore_state(self, state: StateDict) -> None: ...

dory/utils/__init__.py

@@ -0,0 +1,31 @@
+"""Utility modules for Dory SDK."""
+
+from dory.utils.errors import (
+    DoryError,
+    DoryStartupError,
+    DoryShutdownError,
+    DoryStateError,
+    DoryConfigError,
+    DoryK8sError,
+    DoryHealthError,
+    DoryTimeoutError,
+    DoryValidationError,
+)
+from dory.utils.retry import retry_async, RetryConfig
+from dory.utils.timeout import timeout_async, TimeoutConfig
+
+__all__ = [
+    "DoryError",
+    "DoryStartupError",
+    "DoryShutdownError",
+    "DoryStateError",
+    "DoryConfigError",
+    "DoryK8sError",
+    "DoryHealthError",
+    "DoryTimeoutError",
+    "DoryValidationError",
+    "retry_async",
+    "RetryConfig",
+    "timeout_async",
+    "TimeoutConfig",
+]

dory/utils/errors.py

@@ -0,0 +1,59 @@
+"""
+Dory SDK Exception Classes.
+
+All SDK-specific exceptions inherit from DoryError for easy catching.
+"""
+
+
+class DoryError(Exception):
+    """Base exception for all Dory SDK errors."""
+
+    def __init__(self, message: str, cause: Exception | None = None):
+        super().__init__(message)
+        self.message = message
+        self.cause = cause
+
+    def __str__(self) -> str:
+        if self.cause:
+            return f"{self.message}: {self.cause}"
+        return self.message
+
+
+class DoryStartupError(DoryError):
+    """Raised when processor startup fails."""
+    pass
+
+
+class DoryShutdownError(DoryError):
+    """Raised when processor shutdown fails or times out."""
+    pass
+
+
+class DoryStateError(DoryError):
+    """Raised when state operations fail (snapshot, restore, validation)."""
+    pass
+
+
+class DoryConfigError(DoryError):
+    """Raised when configuration is invalid or missing."""
+    pass
+
+
+class DoryK8sError(DoryError):
+    """Raised when Kubernetes API operations fail."""
+    pass
+
+
+class DoryHealthError(DoryError):
+    """Raised when health check operations fail."""
+    pass
+
+
+class DoryTimeoutError(DoryError):
+    """Raised when an operation times out."""
+    pass
+
+
+class DoryValidationError(DoryError):
+    """Raised when validation fails (state schema, config, etc.)."""
+    pass

dory/utils/retry.py

@@ -0,0 +1,115 @@
+"""
+Retry utilities with exponential backoff.
+"""
+
+import asyncio
+import functools
+import random
+from dataclasses import dataclass, field
+from typing import Callable, TypeVar, ParamSpec, Awaitable
+import logging
+
+logger = logging.getLogger(__name__)
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+@dataclass
+class RetryConfig:
+    """Configuration for retry behavior."""
+
+    max_attempts: int = 3
+    base_delay: float = 1.0  # seconds
+    max_delay: float = 60.0  # seconds
+    exponential_base: float = 2.0
+    jitter: bool = True
+    retryable_exceptions: tuple[type[Exception], ...] = field(
+        default_factory=lambda: (Exception,)
+    )
+
+
+def calculate_delay(
+    attempt: int,
+    config: RetryConfig,
+) -> float:
+    """Calculate delay for next retry with exponential backoff."""
+    delay = config.base_delay * (config.exponential_base ** (attempt - 1))
+    delay = min(delay, config.max_delay)
+
+    if config.jitter:
+        # Add random jitter (0-25% of delay)
+        delay = delay * (1 + random.uniform(0, 0.25))
+
+    return delay
+
+
+async def retry_async(
+    func: Callable[P, Awaitable[T]],
+    *args: P.args,
+    config: RetryConfig | None = None,
+    **kwargs: P.kwargs,
+) -> T:
+    """
+    Execute an async function with retry logic.
+
+    Args:
+        func: Async function to execute
+        *args: Positional arguments for func
+        config: Retry configuration (uses defaults if None)
+        **kwargs: Keyword arguments for func
+
+    Returns:
+        Result of func
+
+    Raises:
+        Last exception if all retries fail
+    """
+    if config is None:
+        config = RetryConfig()
+
+    last_exception: Exception | None = None
+
+    for attempt in range(1, config.max_attempts + 1):
+        try:
+            return await func(*args, **kwargs)
+        except config.retryable_exceptions as e:
+            last_exception = e
+
+            if attempt == config.max_attempts:
+                logger.warning(
+                    f"Retry exhausted after {attempt} attempts: {e}"
+                )
+                break
+
+            delay = calculate_delay(attempt, config)
+            logger.debug(
+                f"Retry attempt {attempt}/{config.max_attempts} failed: {e}. "
+                f"Retrying in {delay:.2f}s"
+            )
+            await asyncio.sleep(delay)
+
+    if last_exception:
+        raise last_exception
+
+    # Should never reach here, but satisfy type checker
+    raise RuntimeError("Retry logic error")
+
+
+def with_retry(
+    config: RetryConfig | None = None,
+) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]:
+    """
+    Decorator to add retry logic to async functions.
+
+    Usage:
+        @with_retry(RetryConfig(max_attempts=5))
+        async def my_flaky_function():
+            ...
+    """
+    def decorator(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
+        @functools.wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            return await retry_async(func, *args, config=config, **kwargs)
+        return wrapper
+    return decorator

dory/utils/timeout.py

@@ -0,0 +1,80 @@
+"""
+Timeout utilities for async operations.
+"""
+
+import asyncio
+import functools
+from dataclasses import dataclass
+from typing import Callable, TypeVar, ParamSpec, Awaitable
+
+from dory.utils.errors import DoryTimeoutError
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+@dataclass
+class TimeoutConfig:
+    """Configuration for timeout behavior."""
+
+    timeout_seconds: float
+    error_message: str | None = None
+
+
+async def timeout_async(
+    func: Callable[P, Awaitable[T]],
+    *args: P.args,
+    timeout_seconds: float,
+    error_message: str | None = None,
+    **kwargs: P.kwargs,
+) -> T:
+    """
+    Execute an async function with a timeout.
+
+    Args:
+        func: Async function to execute
+        *args: Positional arguments for func
+        timeout_seconds: Maximum time to wait
+        error_message: Custom error message on timeout
+        **kwargs: Keyword arguments for func
+
+    Returns:
+        Result of func
+
+    Raises:
+        DoryTimeoutError: If function doesn't complete within timeout
+    """
+    try:
+        return await asyncio.wait_for(
+            func(*args, **kwargs),
+            timeout=timeout_seconds,
+        )
+    except asyncio.TimeoutError:
+        msg = error_message or f"Operation timed out after {timeout_seconds}s"
+        raise DoryTimeoutError(msg)
+
+
+def with_timeout(
+    timeout_seconds: float,
+    error_message: str | None = None,
+) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]:
+    """
+    Decorator to add timeout to async functions.
+
+    Usage:
+        @with_timeout(30, "Startup took too long")
+        async def startup():
+            ...
+    """
+    def decorator(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
+        @functools.wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            return await timeout_async(
+                func,
+                *args,
+                timeout_seconds=timeout_seconds,
+                error_message=error_message,
+                **kwargs,
+            )
+        return wrapper
+    return decorator