dory-processor-sdk 0.0.1__py3-none-any.whl

dory/core/processor.py

@@ -0,0 +1,564 @@
+"""
+BaseProcessor - Abstract base class for processor implementations.
+
+Developers implement this class to create their processor applications.
+The SDK handles all lifecycle, state management, and health concerns.
+
+Auto-initialized components (if available):
+- Circuit breakers (self.circuit_breakers)
+- Error classifier (self.error_classifier)
+- OpenTelemetry (self.otel)
+- Request tracker (self.request_tracker)
+- Request ID middleware (self.request_id_middleware)
+- Connection tracker (self.connection_tracker)
+- RabbitMQ publisher (self.publisher)
+
+All handler methods are automatically instrumented via AutoInstrumentMeta.
+"""
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, AsyncIterator, Dict, Any, Optional
+
+from dory.decorators import get_stateful_vars, set_stateful_vars
+from dory.core.meta import AutoInstrumentMeta
+
+if TYPE_CHECKING:
+    from dory.core.context import ExecutionContext
+    from dory.geo import GeoPoint
+
+logger = logging.getLogger(__name__)
+
+
+class BaseProcessor(ABC, metaclass=AutoInstrumentMeta):
+    """
+    Abstract base class for processor implementations.
+
+    Required method:
+    - run(): Main processing loop
+
+    Optional methods (have sensible defaults):
+    - startup(): Initialize resources (default: no-op)
+    - shutdown(): Cleanup resources (default: no-op)
+    - get_state(): Return state dict (default: returns @stateful vars or {})
+    - restore_state(): Restore state (default: restores @stateful vars)
+
+    Optional fault handling hooks:
+    - on_state_restore_failed(): Handle state restore errors
+    - on_rapid_restart_detected(): Handle restart loop
+    - on_health_check_failed(): Handle health check errors
+    - reset_caches(): Clean caches during golden image reset
+
+    Usage:
+        # Minimal implementation (just run method)
+        class MyProcessor(BaseProcessor):
+            counter = stateful(0)
+
+            async def run(self):
+                async for _ in self.run_loop(interval=1):
+                    self.counter += 1
+
+        # Full implementation
+        class MyProcessor(BaseProcessor):
+            async def startup(self):
+                self.model = load_model()
+
+            async def run(self):
+                while not self.context.is_shutdown_requested():
+                    process()
+
+            async def shutdown(self):
+                self.model.close()
+
+            def get_state(self):
+                return {"processed": self.count}
+
+            async def restore_state(self, state):
+                self.count = state.get("processed", 0)
+    """
+
+    # Optional: Define state schema for validation
+    # Schema example: {'processed_count': int, 'last_frame_id': int}
+    state_schema: dict[str, type] | None = None
+
+    # Context is auto-injected by DoryApp (no need to accept in __init__)
+    context: "ExecutionContext"
+
+    # =========================================================================
+    # SDK-Managed Components (auto-initialized with defaults)
+    # Declared here for IDE autocomplete and type checking.
+    # =========================================================================
+    error_classifier: Any
+    circuit_breakers: Dict[str, Any]
+    otel: Optional[Any]
+    request_tracker: Optional[Any]
+    request_id_middleware: Optional[Any]
+    connection_tracker: Optional[Any]
+    publisher: Optional[Any]
+
+    def __init__(self, context: "ExecutionContext | None" = None):
+        """
+        Initialize processor with auto-initialization of SDK components.
+
+        Args:
+            context: ExecutionContext (optional - will be auto-injected if not provided)
+
+        Note:
+            You can override __init__ and call super().__init__(context) to get
+            auto-initialization, or skip super() call to manually initialize.
+        """
+        if context is not None:
+            self.context = context
+
+            # Auto-initialize SDK components if context is available
+            self._auto_initialize_components()
+
+    # =========================================================================
+    # Required Method
+    # =========================================================================
+
+    @abstractmethod
+    async def run(self) -> None:
+        """
+        Main processing loop.
+
+        Called after startup() and restore_state(). Must check
+        context.is_shutdown_requested() periodically to exit gracefully.
+
+        You can use self.run_loop() helper for cleaner code:
+
+            async def run(self):
+                async for _ in self.run_loop(interval=1):
+                    self.counter += 1
+
+        Or traditional while loop:
+
+            async def run(self):
+                while not self.context.is_shutdown_requested():
+                    self.counter += 1
+                    await asyncio.sleep(1)
+
+        Raises:
+            Any exception will cause pod crash
+        """
+        raise NotImplementedError
+
+    # =========================================================================
+    # Optional Lifecycle Methods (Override if needed)
+    # =========================================================================
+
+    async def startup(self) -> None:
+        """
+        Initialize processor resources (optional).
+
+        Called once at pod startup after __init__ but before run().
+        Override to load models, open connections, etc.
+
+        Default: No-op
+        """
+        pass
+
+    async def shutdown(self) -> None:
+        """
+        Cleanup processor resources (optional).
+
+        Called on graceful shutdown (SIGTERM). Has max timeout
+        (configurable via DORY_SHUTDOWN_TIMEOUT_SEC, default 30s).
+        Override to close connections, flush buffers, etc.
+
+        Default: No-op
+        """
+        pass
+
+    def get_state(self) -> dict:
+        """
+        Return state to migrate to next pod (optional).
+
+        Called during migration (must be fast, <1s). State must be
+        JSON-serializable.
+
+        Default: Returns all @stateful decorated attributes, or {} if none.
+
+        Override for custom state:
+            def get_state(self):
+                return {"counter": self.counter, "data": self.data}
+        """
+        # Auto-collect @stateful decorated attributes
+        stateful_state = get_stateful_vars(self)
+        if stateful_state:
+            return stateful_state
+        return {}
+
+    async def restore_state(self, state: dict) -> None:
+        """
+        Restore state from previous pod (optional).
+
+        Called after startup() but before run() if state exists.
+
+        Default: Restores all @stateful decorated attributes from state.
+
+        Override for custom restoration:
+            async def restore_state(self, state):
+                self.counter = state.get("counter", 0)
+        """
+        # Auto-restore @stateful decorated attributes
+        set_stateful_vars(self, state)
+
+    # =========================================================================
+    # Helper Methods
+    # =========================================================================
+
+    async def run_loop(
+        self,
+        interval: float = 1.0,
+        check_migration: bool = True,
+    ) -> AsyncIterator[int]:
+        """
+        Async iterator that yields until shutdown is requested.
+
+        Simplifies the common pattern of checking shutdown in a loop.
+
+        Args:
+            interval: Sleep interval between iterations (seconds)
+            check_migration: If True, also yields when migration is imminent
+
+        Yields:
+            Iteration count (0, 1, 2, ...)
+
+        Usage:
+            async def run(self):
+                async for i in self.run_loop(interval=1):
+                    self.counter += 1
+                    print(f"Iteration {i}")
+
+            # Equivalent to:
+            async def run(self):
+                i = 0
+                while not self.context.is_shutdown_requested():
+                    self.counter += 1
+                    print(f"Iteration {i}")
+                    i += 1
+                    await asyncio.sleep(1)
+        """
+        iteration = 0
+        while not self.context.is_shutdown_requested():
+            yield iteration
+            iteration += 1
+
+            # Check if migration is imminent
+            if check_migration and self.context.is_migration_imminent():
+                self.context.logger().info(
+                    f"Migration imminent, completing iteration {iteration}"
+                )
+
+            await asyncio.sleep(interval)
+
+    def is_shutting_down(self) -> bool:
+        """
+        Convenience method to check if shutdown is requested.
+
+        Returns:
+            True if shutdown has been requested
+        """
+        return self.context.is_shutdown_requested()
+
+    async def publish(
+        self,
+        event_type: str,
+        location: "GeoPoint",
+        payload: dict[str, Any],
+        *,
+        headers: dict[str, Any] | None = None,
+        exchange: str | None = None,
+    ) -> None:
+        """Publish a single message with a geo location.
+
+        The developer provides the event type, a
+        :class:`~dory.geo.GeoPoint` (e.g. from
+        :meth:`CameraGeolocalizer.estimate`), and a payload already in
+        the unified envelope format.  The SDK converts the coordinates
+        to a geohash-based routing key and publishes the payload as-is
+        (no additional envelope wrapping).
+
+        Examples::
+
+            loc = geo.estimate(box)
+            await self.publish(
+                "accident",
+                location=loc,
+                payload=MessageEnvelope(payload={...}).to_dict(),
+            )
+
+        Args:
+            event_type: Event type for routing (e.g., "accident", "detection").
+            location: A :class:`~dory.geo.GeoPoint` with ``.lat`` and
+                ``.lng`` attributes.
+            payload: Message payload in unified envelope format.
+            headers: Optional AMQP message headers.
+            exchange: Optional exchange override.
+
+        Raises:
+            RuntimeError: If publisher is not initialized.
+            ValueError: If event_type is empty or coordinates are out of range.
+        """
+        if self.publisher is None:
+            raise RuntimeError(
+                "RabbitMQ publisher not initialized. "
+                "Set DORY_RABBITMQ_OAUTH2_TOKEN_URL and OAuth2 credentials to enable."
+            )
+
+        from dory.output.routing import build_routing_key_from_geo
+
+        routing_key = build_routing_key_from_geo(
+            event_type, location.lat, location.lng
+        )
+        logger.info(
+            "Publishing message event_type=%s lat=%.6f lon=%.6f routing_key=%s",
+            event_type,
+            location.lat,
+            location.lng,
+            routing_key,
+        )
+
+        await self.publisher.publish(
+            routing_key=routing_key,
+            data=payload,
+            exchange=exchange,
+            headers=headers,
+            raw=True,
+        )
+
+    # =========================================================================
+    # Optional Fault Handling Hooks
+    # =========================================================================
+
+    async def on_state_restore_failed(self, error: Exception) -> bool:
+        """
+        Called if state restore fails.
+
+        Override to attempt recovery (e.g., fetch from external backup).
+        Return True to start with golden image, False to exit and crash.
+
+        Args:
+            error: Exception from restore_state() or validation
+
+        Returns:
+            True to continue with golden image, False to exit
+        """
+        return True  # Default: continue with golden image
+
+    async def on_rapid_restart_detected(self, restart_count: int) -> bool:
+        """
+        Called if restart loop detected (3+ restarts in 5 minutes).
+
+        Override to attempt recovery (e.g., reinitialize state, reset
+        connections). Return True to continue, False to trigger golden reset.
+
+        Args:
+            restart_count: Number of restarts detected
+
+        Returns:
+            True to continue, False to force golden reset
+        """
+        return True  # Default: continue (SDK will start golden)
+
+    async def on_health_check_failed(self, error: Exception) -> bool:
+        """
+        Called if health check fails.
+
+        Override to attempt recovery (e.g., reconnect to external services).
+        Return True to retry health check, False to fail.
+
+        Args:
+            error: Exception from health check
+
+        Returns:
+            True to retry, False to fail
+        """
+        return False  # Default: fail health check
+
+    def reset_caches(self) -> None:
+        """
+        Called during golden image reset.
+
+        Override to clear any in-memory caches, buffers, or temporary
+        state that should not persist through a golden reset.
+        """
+        pass  # Default: no caches to reset
+
+    # =========================================================================
+    # Auto-Initialization
+    # =========================================================================
+
+    def _auto_initialize_components(self) -> None:
+        """
+        Auto-initialize SDK components with sensible defaults.
+
+        Called automatically during __init__ if context is available.
+        Each component is optional — if its package is not installed,
+        initialization is silently skipped.
+        """
+        if not hasattr(self, "context") or self.context is None:
+            logger.debug("Context not available, skipping auto-initialization")
+            return
+
+        self._init_error_classifier()
+        self._init_circuit_breakers()
+        self._init_opentelemetry()
+        self._init_request_tracking()
+        self._init_request_id()
+        self._init_connection_tracking()
+        self._init_publisher()
+
+        logger.debug("Auto-initialization complete")
+
+    def _init_error_classifier(self) -> None:
+        """Initialize error classifier."""
+        try:
+            from dory.errors import ErrorClassifier
+            self.error_classifier = ErrorClassifier()
+        except ImportError:
+            self.error_classifier = None
+
+    def _init_circuit_breakers(self) -> None:
+        """Create default circuit breakers for common services."""
+        self.circuit_breakers = {}
+
+        try:
+            from dory.resilience import CircuitBreaker
+        except ImportError:
+            return
+
+        for name in ("database", "external_api", "cache"):
+            self.circuit_breakers[name] = CircuitBreaker(
+                name=name,
+                failure_threshold=5,
+                success_threshold=2,
+                timeout_seconds=30.0,
+                half_open_max_calls=3,
+            )
+
+    def _init_opentelemetry(self) -> None:
+        """Initialize OpenTelemetry with defaults.
+
+        Uses DORY_APP_VERSION env var (injected by orchestrator from DB) for
+        service.version. Falls back to "1.0.0" if not set.
+        """
+        self.otel = None
+
+        try:
+            import os
+            from dory.monitoring import OpenTelemetryManager
+            service_version = os.environ.get("DORY_APP_VERSION", "1.0.0")
+            self.otel = OpenTelemetryManager(
+                service_name="dory-app",
+                service_version=service_version,
+                environment="production",
+                console_export=True,
+            )
+            self.otel.initialize()
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to initialize OpenTelemetry: {e}")
+
+    def _init_request_tracking(self) -> None:
+        """Initialize request tracking."""
+        self.request_tracker = None
+
+        try:
+            from dory.middleware import RequestTracker
+            self.request_tracker = RequestTracker(
+                max_history=1000,
+                enable_history=True,
+            )
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to initialize request tracking: {e}")
+
+    def _init_request_id(self) -> None:
+        """Initialize request ID middleware."""
+        self.request_id_middleware = None
+
+        try:
+            from dory.middleware import RequestIdMiddleware
+            self.request_id_middleware = RequestIdMiddleware(
+                header_name="X-Request-ID",
+                log_request_id=True,
+            )
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to initialize request ID middleware: {e}")
+
+    def _init_connection_tracking(self) -> None:
+        """Initialize connection tracking."""
+        self.connection_tracker = None
+
+        try:
+            from dory.middleware import ConnectionTracker
+            self.connection_tracker = ConnectionTracker()
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to initialize connection tracking: {e}")
+
+    def _init_publisher(self) -> None:
+        """Initialize OAuth2-authenticated RabbitMQ publisher.
+
+        The publisher is created when DORY_RABBITMQ_OAUTH2_TOKEN_URL is set.
+        All configuration is read from environment variables.
+        """
+        self.publisher = None
+
+        import os
+        token_url = os.environ.get("DORY_RABBITMQ_OAUTH2_TOKEN_URL", "")
+        if not token_url:
+            return
+
+        try:
+            from dory.output.rabbitmq import RabbitMQPublisher, PublisherConfig
+        except ImportError:
+            return
+
+        try:
+            from dory.output.formatter import JSONFormatter
+            from dory.auth.oauth2 import OAuth2TokenProvider
+
+            publisher_config = PublisherConfig(
+                url="",  # populated by url_provider
+                exchange=os.environ.get("DORY_RABBITMQ_EXCHANGE", "dory.output"),
+                exchange_type="topic",
+                durable=True,
+            )
+
+            token_provider = OAuth2TokenProvider(
+                token_url=token_url,
+                client_id=os.environ.get("DORY_RABBITMQ_OAUTH2_CLIENT_ID", ""),
+                client_secret=os.environ.get("DORY_RABBITMQ_OAUTH2_CLIENT_SECRET", ""),
+                scopes=["rabbitmq/write:all"],
+            )
+            _host = os.environ.get("DORY_RABBITMQ_HOST", "")
+            _port = 5671
+            _vhost = os.environ.get("DORY_RABBITMQ_VHOST", "/")
+            _tls = os.environ.get("DORY_RABBITMQ_TLS_ENABLED", "true").lower() == "true"
+
+            async def url_provider(
+                _tp=token_provider, _h=_host, _p=_port, _v=_vhost, _t=_tls
+            ):
+                return await _tp.build_amqp_url(
+                    host=_h, port=_p, vhost=_v, tls=_t
+                )
+
+            self.publisher = RabbitMQPublisher(
+                config=publisher_config,
+                formatter=JSONFormatter(),
+                url_provider=url_provider,
+            )
+            logger.info("RabbitMQ publisher initialized (OAuth2)")
+
+        except Exception as e:
+            logger.warning(f"Failed to initialize RabbitMQ publisher: {e}")
+            self.publisher = None

dory/core/signals.py

@@ -0,0 +1,122 @@
+"""
+SignalHandler - Handles OS signals for graceful shutdown.
+
+Captures SIGTERM, SIGINT, and SIGUSR1 and triggers appropriate
+actions in the SDK.
+"""
+
+import asyncio
+import logging
+import signal
+import sys
+from typing import Callable, Awaitable
+
+logger = logging.getLogger(__name__)
+
+
+class SignalHandler:
+    """
+    Handles OS signals for graceful shutdown.
+
+    Signals handled:
+        SIGTERM: Graceful shutdown (from Kubelet)
+        SIGINT: Graceful shutdown (Ctrl+C for local testing)
+        SIGUSR1: Trigger state snapshot (for debugging)
+    """
+
+    def __init__(self):
+        self._shutdown_callback: Callable[[], Awaitable[None]] | None = None
+        self._snapshot_callback: Callable[[], Awaitable[None]] | None = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._shutdown_triggered = False
+
+    def setup(
+        self,
+        shutdown_callback: Callable[[], Awaitable[None]],
+        snapshot_callback: Callable[[], Awaitable[None]] | None = None,
+    ) -> None:
+        """
+        Setup signal handlers.
+
+        Args:
+            shutdown_callback: Async callback for graceful shutdown
+            snapshot_callback: Optional async callback for state snapshot
+        """
+        self._shutdown_callback = shutdown_callback
+        self._snapshot_callback = snapshot_callback
+        self._loop = asyncio.get_event_loop()
+
+        # Register signal handlers
+        if sys.platform != "win32":
+            # Unix signals
+            self._loop.add_signal_handler(
+                signal.SIGTERM,
+                self._handle_shutdown_signal,
+                "SIGTERM",
+            )
+            self._loop.add_signal_handler(
+                signal.SIGINT,
+                self._handle_shutdown_signal,
+                "SIGINT",
+            )
+            self._loop.add_signal_handler(
+                signal.SIGUSR1,
+                self._handle_snapshot_signal,
+            )
+            logger.debug("Signal handlers registered (Unix)")
+        else:
+            # Windows - limited signal support
+            signal.signal(signal.SIGTERM, self._handle_shutdown_signal_sync)
+            signal.signal(signal.SIGINT, self._handle_shutdown_signal_sync)
+            logger.debug("Signal handlers registered (Windows)")
+
+    def _handle_shutdown_signal(self, sig_name: str) -> None:
+        """Handle SIGTERM/SIGINT asynchronously."""
+        if self._shutdown_triggered:
+            logger.warning(f"Received {sig_name} but shutdown already in progress")
+            return
+
+        self._shutdown_triggered = True
+        logger.info(f"Received {sig_name}, initiating graceful shutdown")
+
+        if self._shutdown_callback and self._loop:
+            asyncio.ensure_future(
+                self._shutdown_callback(),
+                loop=self._loop,
+            )
+
+    def _handle_shutdown_signal_sync(self, signum: int, frame) -> None:
+        """Handle signal synchronously (Windows compatibility)."""
+        sig_name = signal.Signals(signum).name
+        self._handle_shutdown_signal(sig_name)
+
+    def _handle_snapshot_signal(self) -> None:
+        """Handle SIGUSR1 for debug state snapshot."""
+        logger.info("Received SIGUSR1, triggering state snapshot")
+
+        if self._snapshot_callback and self._loop:
+            asyncio.ensure_future(
+                self._snapshot_callback(),
+                loop=self._loop,
+            )
+
+    def remove_handlers(self) -> None:
+        """Remove signal handlers during shutdown."""
+        if self._loop and sys.platform != "win32":
+            try:
+                self._loop.remove_signal_handler(signal.SIGTERM)
+                self._loop.remove_signal_handler(signal.SIGINT)
+                self._loop.remove_signal_handler(signal.SIGUSR1)
+                logger.debug("Signal handlers removed")
+            except (ValueError, RuntimeError):
+                # Handler not registered or loop closed
+                pass
+
+    @property
+    def shutdown_triggered(self) -> bool:
+        """Check if shutdown has been triggered."""
+        return self._shutdown_triggered
+
+    def reset(self) -> None:
+        """Reset shutdown state (for testing)."""
+        self._shutdown_triggered = False