dory-processor-sdk 0.0.1__py3-none-any.whl

dory/output/routing.py

@@ -0,0 +1,318 @@
+"""Routing key builder utilities for topic-based message routing."""
+
+import logging
+import os
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Base-32 alphabet used by standard geohash encoding
+_GEOHASH_ALPHABET = "0123456789bcdefghjkmnpqrstuvwxyz"
+
+
+def _encode_geohash(latitude: float, longitude: float, precision: int = 9) -> str:
+    """Encode latitude/longitude into a geohash string.
+
+    Uses the standard base-32 geohash algorithm (no external dependency).
+
+    Args:
+        latitude: Latitude in decimal degrees (-90 to 90).
+        longitude: Longitude in decimal degrees (-180 to 180).
+        precision: Number of characters in the geohash (default 9).
+
+    Returns:
+        Geohash string of the requested precision.
+    """
+    lat_range = (-90.0, 90.0)
+    lon_range = (-180.0, 180.0)
+    bits = 0
+    char_index = 0
+    is_lon = True
+    result: list[str] = []
+
+    while len(result) < precision:
+        if is_lon:
+            mid = (lon_range[0] + lon_range[1]) / 2
+            if longitude >= mid:
+                char_index = (char_index << 1) | 1
+                lon_range = (mid, lon_range[1])
+            else:
+                char_index = char_index << 1
+                lon_range = (lon_range[0], mid)
+        else:
+            mid = (lat_range[0] + lat_range[1]) / 2
+            if latitude >= mid:
+                char_index = (char_index << 1) | 1
+                lat_range = (mid, lat_range[1])
+            else:
+                char_index = char_index << 1
+                lat_range = (lat_range[0], mid)
+
+        is_lon = not is_lon
+        bits += 1
+
+        if bits == 5:
+            result.append(_GEOHASH_ALPHABET[char_index])
+            bits = 0
+            char_index = 0
+
+    return "".join(result)
+
+
+def _resolve_geohash(precision: int = 9) -> Optional[str]:
+    """Resolve geohash from environment variables.
+
+    Checks in order:
+        1. ``DORY_GEOHASH`` - direct geohash string (auto-injected by orchestrator
+           from sensors.location_point)
+        2. ``DORY_LATITUDE`` + ``DORY_LONGITUDE`` - computed from coordinates
+           (local development fallback)
+
+    Args:
+        precision: Geohash precision when computing from lat/lon (default 9).
+
+    Returns:
+        Geohash string, or None if not configured.
+    """
+    # Option 1: direct geohash (auto-injected by orchestrator from sensors.location_point)
+    geohash = os.environ.get("DORY_GEOHASH", "").strip()
+    if geohash:
+        return geohash.lower()
+
+    # Option 2: compute from lat/lon (local development fallback)
+    lat_str = os.environ.get("DORY_LATITUDE", "").strip()
+    lon_str = os.environ.get("DORY_LONGITUDE", "").strip()
+    if lat_str and lon_str:
+        try:
+            lat = float(lat_str)
+            lon = float(lon_str)
+        except ValueError:
+            logger.warning(
+                "DORY_LATITUDE ('%s') and DORY_LONGITUDE ('%s') "
+                "must be valid decimal numbers — geohash disabled",
+                lat_str,
+                lon_str,
+            )
+            return None
+        return _encode_geohash(lat, lon, precision)
+
+    return None
+
+
+# Sentinel: distinguishes "not yet resolved" from "resolved to None"
+_UNRESOLVED = object()
+
+# Cached geohash — resolved once per process
+_cached_geohash: object = _UNRESOLVED
+
+# Cached processor_id — resolved once per process
+_cached_processor_id: object = _UNRESOLVED
+
+
+def _resolve_processor_id() -> Optional[str]:
+    """Resolve processor_id from the ``PROCESSOR_ID`` environment variable.
+
+    Returns:
+        Processor ID string, or None if not configured.
+    """
+    pid = os.environ.get("PROCESSOR_ID", "").strip()
+    if pid:
+        return pid
+    return None
+
+
+def get_processor_id() -> Optional[str]:
+    """Get the processor_id for this deployment, resolved from environment.
+
+    The result is cached after the first call. Returns None if not configured.
+
+    In production the orchestrator auto-injects ``PROCESSOR_ID`` from
+    ``processors.id``. For local development you can set ``PROCESSOR_ID``
+    manually.
+
+    Returns:
+        Processor ID string, or None if not configured.
+    """
+    global _cached_processor_id
+    if _cached_processor_id is _UNRESOLVED:
+        _cached_processor_id = _resolve_processor_id()
+        if _cached_processor_id:
+            logger.info("Processor ID resolved: %s", _cached_processor_id)
+        else:
+            logger.warning(
+                "No processor_id configured (PROCESSOR_ID not set). "
+                "Routing keys will not include processor_id prefix."
+            )
+    return _cached_processor_id
+
+
+def get_geohash(precision: int = 9) -> Optional[str]:
+    """Get the geohash for this deployment, resolved from environment.
+
+    The result is cached after the first call. Returns None if no geohash
+    is configured (sensor has no location_point).
+
+    In production the orchestrator auto-injects ``DORY_GEOHASH`` from the
+    sensor's ``location_point`` column. For local development you can set
+    ``DORY_GEOHASH`` or ``DORY_LATITUDE`` + ``DORY_LONGITUDE`` manually.
+
+    Args:
+        precision: Geohash precision when computing from lat/lon.
+
+    Returns:
+        Geohash string, or None if not configured.
+    """
+    global _cached_geohash
+    if _cached_geohash is _UNRESOLVED:
+        _cached_geohash = _resolve_geohash(precision)
+        if _cached_geohash:
+            logger.info("Geohash resolved: %s", _cached_geohash)
+        else:
+            logger.warning(
+                "No geohash configured (DORY_GEOHASH not set). "
+                "Routing keys will not include geohash segments."
+            )
+    return _cached_geohash
+
+
+def build_routing_key_from_geo(
+    event_type: str,
+    latitude: float,
+    longitude: float,
+    *,
+    precision: int = 9,
+    segment_length: int = 1,
+) -> str:
+    """Build a routing key from event type and explicit geo coordinates.
+
+    Converts the latitude/longitude to a geohash and appends it as
+    dot-separated segments to the event type. If ``DORY_PROCESSOR_ID``
+    is set, it is prepended as the first segment.
+
+    Produces: ``<processor_id>.<event_type>.<geohash_segments...>`` (with processor_id)
+    Produces: ``<event_type>.<geohash_segments...>`` (without processor_id)
+
+    Examples:
+        >>> build_routing_key_from_geo("accident", 39.1, -84.5)
+        'accident.d.h.z.0.6.m.b.8.e'
+
+    Args:
+        event_type: Event type (e.g., "accident", "detection").
+        latitude: Latitude in decimal degrees (-90 to 90).
+        longitude: Longitude in decimal degrees (-180 to 180).
+        precision: Geohash precision (default 9).
+        segment_length: Geohash characters per routing segment (default 1).
+
+    Returns:
+        Dot-separated routing key.
+
+    Raises:
+        ValueError: If event_type is empty, segment_length < 1, or
+            coordinates are out of range.
+    """
+    event_type = event_type.strip()
+    if not event_type:
+        raise ValueError("event_type must not be empty")
+    if segment_length < 1:
+        raise ValueError("segment_length must be >= 1")
+    if not (-90.0 <= latitude <= 90.0):
+        raise ValueError(f"latitude must be between -90 and 90, got {latitude}")
+    if not (-180.0 <= longitude <= 180.0):
+        raise ValueError(f"longitude must be between -180 and 180, got {longitude}")
+
+    geohash = _encode_geohash(latitude, longitude, precision)
+    segments = [
+        geohash[i : i + segment_length]
+        for i in range(0, len(geohash), segment_length)
+    ]
+    processor_id = get_processor_id()
+    parts = [processor_id, event_type, *segments] if processor_id else [event_type, *segments]
+    key = ".".join(parts)
+    logger.debug(
+        "build_routing_key_from_geo: processor_id=%s event_type=%s lat=%.6f lon=%.6f geohash=%s routing_key=%s",
+        processor_id,
+        event_type,
+        latitude,
+        longitude,
+        geohash,
+        key,
+    )
+    return key
+
+
+def build_routing_key(
+    event_type: str,
+    segment_length: int = 1,
+) -> str:
+    """Build a routing key from just an event type.
+
+    The processor_id is automatically resolved from ``PROCESSOR_ID``
+    (auto-injected by the orchestrator from ``processors.id``). The geohash
+    is resolved from ``DORY_GEOHASH`` (auto-injected from
+    ``sensors.location_point``). Both prefixes are optional — if either is
+    missing the routing key omits that segment.
+
+    Produces (all present):    ``<processor_id>.<event_type>.<geohash_segments...>``
+    Produces (no processor):   ``<event_type>.<geohash_segments...>``
+    Produces (no geohash):     ``<processor_id>.<event_type>``
+    Produces (neither):        ``<event_type>``
+
+    Examples:
+        # With PROCESSOR_ID=abc-123 and DORY_GEOHASH=dhz06mb8e
+        >>> build_routing_key("accident")
+        'abc-123.accident.d.h.z.0.6.m.b.8.e'
+
+        # With only DORY_GEOHASH=dhz06mb8e
+        >>> build_routing_key("accident")
+        'accident.d.h.z.0.6.m.b.8.e'
+
+        # With neither (local dev, processor not in k8s)
+        >>> build_routing_key("accident")
+        'accident'
+
+    Subscriber binding examples (when processor_id is in the key)::
+
+        *.accident.#                 -> all accidents from any processor
+        <pid>.accident.#             -> accidents from one specific processor
+        *.accident.d.h.z.#           -> accidents in geohash prefix "dhz"
+        *.*.d.h.z.0.6.#              -> any event in a narrower area
+
+    Args:
+        event_type: Event type (e.g., "accident", "detection", "alert").
+        segment_length: Geohash characters per routing segment (default 1).
+
+    Returns:
+        Dot-separated routing key.
+
+    Raises:
+        ValueError: If event_type is empty or segment_length < 1.
+    """
+    event_type = event_type.strip()
+    if not event_type:
+        raise ValueError("event_type must not be empty")
+
+    if segment_length < 1:
+        raise ValueError("segment_length must be >= 1")
+
+    processor_id = get_processor_id()
+    geohash = get_geohash()
+
+    parts: list[str] = []
+    if processor_id:
+        parts.append(processor_id)
+    parts.append(event_type)
+    if geohash:
+        parts.extend(
+            geohash[i : i + segment_length]
+            for i in range(0, len(geohash), segment_length)
+        )
+
+    key = ".".join(parts)
+    logger.debug(
+        "build_routing_key: processor_id=%s event_type=%s geohash=%s routing_key=%s",
+        processor_id,
+        event_type,
+        geohash,
+        key,
+    )
+    return key

dory/output/validator.py

@@ -0,0 +1,199 @@
+"""
+Envelope validator for subscriber-side schema version dispatch.
+
+Provides version-aware message handling so subscribers can process
+messages from publishers running different schema versions.
+
+Usage:
+    validator = EnvelopeValidator()
+
+    # Register handlers per schema version
+    validator.register("0.1", handle_v0)
+    validator.register("1.0", handle_v1)
+
+    # Process incoming message
+    result = await validator.handle(raw_message_dict)
+"""
+
+import logging
+from typing import Any, Callable, Awaitable
+
+from dory.output.envelope import (
+    MessageEnvelope,
+    ENVELOPE_SCHEMA_VERSION,
+)
+
+logger = logging.getLogger(__name__)
+
+# Type for async handler functions
+HandlerFunc = Callable[[MessageEnvelope], Awaitable[Any]]
+
+# Type for sync handler functions
+SyncHandlerFunc = Callable[[MessageEnvelope], Any]
+
+
+class UnsupportedVersionError(Exception):
+    """Raised when no handler is registered for a schema version."""
+
+    def __init__(self, version: str, available: list[str]):
+        self.version = version
+        self.available = available
+        super().__init__(
+            f"No handler for schema_version={version}. "
+            f"Available: {available}"
+        )
+
+
+class EnvelopeValidator:
+    """Subscriber-side envelope validator with version dispatch.
+
+    Validates incoming envelopes and routes to the appropriate handler
+    based on schema_version.
+
+    Version matching strategy:
+    1. Exact match (e.g., "0.1" -> handler for "0.1")
+    2. Major version fallback (e.g., "0.3" -> handler for "0.1")
+    3. UnsupportedVersionError if no match
+
+    Args:
+        strict_version: If True, require exact version match (no fallback).
+    """
+
+    def __init__(
+        self,
+        strict_version: bool = False,
+        **kwargs: Any,
+    ):
+        self._handlers: dict[str, HandlerFunc] = {}
+        self._sync_handlers: dict[str, SyncHandlerFunc] = {}
+        self._strict_version = strict_version
+
+    def register(
+        self,
+        version: str,
+        handler: HandlerFunc | SyncHandlerFunc,
+        is_async: bool = True,
+    ) -> None:
+        """Register a handler for a schema version.
+
+        Args:
+            version: Schema version string (e.g., "0.1", "1.0").
+            handler: Async or sync callable that receives a MessageEnvelope.
+            is_async: Whether the handler is async (default True).
+        """
+        if is_async:
+            self._handlers[version] = handler  # type: ignore[assignment]
+        else:
+            self._sync_handlers[version] = handler  # type: ignore[assignment]
+        logger.debug(f"Registered {'async' if is_async else 'sync'} handler for v{version}")
+
+    def unregister(self, version: str) -> None:
+        """Remove a handler for a schema version."""
+        self._handlers.pop(version, None)
+        self._sync_handlers.pop(version, None)
+
+    @property
+    def registered_versions(self) -> list[str]:
+        """List all registered version strings."""
+        versions = set(self._handlers.keys()) | set(self._sync_handlers.keys())
+        return sorted(versions)
+
+    def validate(self, data: dict[str, Any]) -> MessageEnvelope:
+        """Validate and parse raw message data into a MessageEnvelope.
+
+        Args:
+            data: Raw dictionary from deserialized message.
+
+        Returns:
+            Parsed MessageEnvelope.
+
+        Raises:
+            ValueError: If required envelope fields are missing.
+        """
+        # Check for envelope structure
+        if "schema_version" not in data:
+            raise ValueError("Missing required field: schema_version")
+        if "payload" not in data:
+            raise ValueError("Missing required field: payload")
+
+        envelope = MessageEnvelope.from_dict(data)
+
+        return envelope
+
+    def resolve_handler(self, version: str) -> tuple[Any, bool]:
+        """Find the best matching handler for a version.
+
+        Args:
+            version: Schema version string.
+
+        Returns:
+            Tuple of (handler, is_async).
+
+        Raises:
+            UnsupportedVersionError: If no handler matches.
+        """
+        # 1. Exact match in async handlers
+        if version in self._handlers:
+            return self._handlers[version], True
+
+        # 2. Exact match in sync handlers
+        if version in self._sync_handlers:
+            return self._sync_handlers[version], False
+
+        # 3. Major version fallback (if not strict)
+        if not self._strict_version:
+            try:
+                major = version.split(".")[0]
+                major_key = f"{major}.0"
+
+                if major_key in self._handlers:
+                    logger.debug(
+                        f"Using major version fallback: v{version} -> v{major_key}"
+                    )
+                    return self._handlers[major_key], True
+
+                if major_key in self._sync_handlers:
+                    logger.debug(
+                        f"Using major version fallback: v{version} -> v{major_key}"
+                    )
+                    return self._sync_handlers[major_key], False
+            except (ValueError, IndexError):
+                pass
+
+        raise UnsupportedVersionError(version, self.registered_versions)
+
+    async def handle(self, data: dict[str, Any]) -> Any:
+        """Validate an envelope and dispatch to the registered handler.
+
+        Args:
+            data: Raw dictionary from deserialized message.
+
+        Returns:
+            Result from the handler.
+
+        Raises:
+            ValueError: If envelope is invalid.
+            UnsupportedVersionError: If no handler for the version.
+        """
+        envelope = self.validate(data)
+        handler, is_async = self.resolve_handler(envelope.schema_version)
+
+        if is_async:
+            return await handler(envelope)
+        else:
+            return handler(envelope)
+
+    def can_handle(self, version: str) -> bool:
+        """Check if a handler exists for this version (including fallback).
+
+        Args:
+            version: Schema version string.
+
+        Returns:
+            True if a handler would be found for this version.
+        """
+        try:
+            self.resolve_handler(version)
+            return True
+        except UnsupportedVersionError:
+            return False

dory/py.typed

@@ -0,0 +1,2 @@
+# Marker file for PEP 561
+# This file indicates that the dory package supports type checking

dory/recovery/__init__.py

@@ -0,0 +1,60 @@
+"""Recovery and fault handling modules."""
+
+from dory.recovery.restart_detector import RestartDetector, RestartInfo
+from dory.recovery.state_validator import StateValidator
+from dory.recovery.golden_image import GoldenImageManager, ResetLevel, ResetResult, CacheResetManager
+from dory.recovery.recovery_decision import RecoveryDecisionMaker, RecoveryDecision
+from dory.recovery.golden_snapshot import (
+    GoldenSnapshotManager,
+    Snapshot,
+    SnapshotMetadata,
+    SnapshotStorageError,
+    SnapshotValidationError,
+    SnapshotFormat,
+)
+from dory.recovery.golden_validator import (
+    GoldenValidator,
+    ValidationResult,
+    ValidationIssue,
+    ValidationSeverity,
+)
+from dory.recovery.partial_recovery import (
+    PartialRecoveryManager,
+    RecoveryResult as PartialRecoveryResult,
+    FieldRecovery,
+    FieldStatus,
+    numeric_recovery_strategy,
+    string_recovery_strategy,
+    list_recovery_strategy,
+    dict_recovery_strategy,
+)
+
+__all__ = [
+    "RestartDetector",
+    "RestartInfo",
+    "StateValidator",
+    "GoldenImageManager",
+    "ResetLevel",
+    "ResetResult",
+    "CacheResetManager",
+    "RecoveryDecisionMaker",
+    "RecoveryDecision",
+    "GoldenSnapshotManager",
+    "Snapshot",
+    "SnapshotMetadata",
+    "SnapshotStorageError",
+    "SnapshotValidationError",
+    "SnapshotFormat",
+    "GoldenValidator",
+    "ValidationResult",
+    "ValidationIssue",
+    "ValidationSeverity",
+    "PartialRecoveryManager",
+    "PartialRecoveryResult",
+    "FieldRecovery",
+    "FieldStatus",
+    "numeric_recovery_strategy",
+    "string_recovery_strategy",
+    "list_recovery_strategy",
+    "dict_recovery_strategy",
+]