edda-framework 0.1.0__py3-none-any.whl

edda/outbox/relayer.py

@@ -0,0 +1,274 @@
+"""
+Outbox Relayer - Background publisher for transactional outbox pattern.
+
+This module implements a background task that polls the database for pending
+outbox events and publishes them to a Message Broker as CloudEvents.
+"""
+
+import asyncio
+import contextlib
+import logging
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
+
+import httpx
+from cloudevents.conversion import to_structured
+from cloudevents.http import CloudEvent
+
+if TYPE_CHECKING:
+    from edda.storage.protocol import StorageProtocol
+
+logger = logging.getLogger(__name__)
+
+
+class OutboxRelayer:
+    """
+    Background relayer for publishing outbox events.
+
+    The relayer polls the database for pending events and publishes them
+    to a Message Broker. It implements exponential backoff for retries
+    and graceful shutdown.
+
+    Example:
+        >>> storage = SQLiteStorage("saga.db")
+        >>> relayer = OutboxRelayer(
+        ...     storage=storage,
+        ...     broker_url="http://broker-ingress.svc.cluster.local/default/default",
+        ...     poll_interval=1.0,
+        ...     max_retries=3
+        ... )
+        >>> await relayer.start()
+    """
+
+    def __init__(
+        self,
+        storage: "StorageProtocol",
+        broker_url: str,
+        poll_interval: float = 1.0,
+        max_retries: int = 3,
+        batch_size: int = 10,
+        max_age_hours: float | None = None,
+    ):
+        """
+        Initialize the Outbox Relayer.
+
+        Args:
+            storage: Storage backend for outbox events
+            broker_url: Message Broker URL for publishing events
+            poll_interval: Polling interval in seconds (default: 1.0)
+            max_retries: Maximum retry attempts (default: 3)
+            batch_size: Number of events to process per batch (default: 10)
+            max_age_hours: Maximum event age in hours before expiration (default: None, disabled)
+                          Events older than this are marked as 'expired' and won't be retried.
+        """
+        self.storage = storage
+        self.broker_url = broker_url
+        self.poll_interval = poll_interval
+        self.max_retries = max_retries
+        self.batch_size = batch_size
+        self.max_age_hours = max_age_hours
+
+        self._task: asyncio.Task[Any] | None = None
+        self._running = False
+        self._http_client: httpx.AsyncClient | None = None
+
+    async def start(self) -> None:
+        """
+        Start the background relayer task.
+
+        This creates an HTTP client and starts the polling loop in a background task.
+        """
+        if self._running:
+            logger.warning("Outbox relayer is already running")
+            return
+
+        self._running = True
+        self._http_client = httpx.AsyncClient(timeout=30.0)
+
+        # Start background task
+        self._task = asyncio.create_task(self._poll_loop())
+        logger.info(
+            f"Outbox relayer started (broker={self.broker_url}, "
+            f"poll_interval={self.poll_interval}s)"
+        )
+
+    async def stop(self) -> None:
+        """
+        Stop the background relayer task gracefully.
+
+        This cancels the polling loop and closes the HTTP client.
+        """
+        if not self._running:
+            return
+
+        self._running = False
+
+        # Cancel background task
+        if self._task:
+            self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._task
+
+        # Close HTTP client
+        if self._http_client:
+            await self._http_client.aclose()
+
+        logger.info("Outbox relayer stopped")
+
+    async def _poll_loop(self) -> None:
+        """
+        Main polling loop.
+
+        Continuously polls the database for pending events and publishes them.
+        """
+        while self._running:
+            try:
+                await self._poll_and_publish()
+            except Exception as e:
+                logger.error(f"Error in outbox relayer poll loop: {e}")
+
+            # Wait before next poll
+            await asyncio.sleep(self.poll_interval)
+
+    async def _poll_and_publish(self) -> None:
+        """
+        Poll for pending events and publish them.
+
+        Fetches a batch of pending events from the database and attempts
+        to publish each one to the Message Broker.
+        """
+        # Get pending events
+        events = await self.storage.get_pending_outbox_events(limit=self.batch_size)
+
+        if not events:
+            return
+
+        logger.debug(f"Processing {len(events)} pending outbox events")
+
+        # Publish each event
+        for event in events:
+            try:
+                await self._publish_event(event)
+            except Exception as e:
+                logger.error(
+                    f"Failed to publish event {event['event_id']}: {e}",
+                    exc_info=True,
+                )
+
+    async def _publish_event(self, event: dict[str, Any]) -> None:
+        """
+        Publish a single outbox event to the Message Broker.
+
+        Args:
+            event: Outbox event record from database
+
+        Raises:
+            Exception: If publishing fails after max retries
+        """
+        event_id = event["event_id"]
+        retry_count = event["retry_count"]
+
+        # Check if max age exceeded (optional feature)
+        if self.max_age_hours is not None:
+            created_at = event["created_at"]
+            # Convert to datetime if it's a string (from database)
+            if isinstance(created_at, str):
+                created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+            elif not hasattr(created_at, "tzinfo") or created_at.tzinfo is None:
+                # Assume UTC if no timezone info
+                created_at = created_at.replace(tzinfo=UTC)
+
+            event_age = datetime.now(UTC) - created_at
+            max_age = timedelta(hours=self.max_age_hours)
+
+            if event_age > max_age:
+                logger.warning(
+                    f"Event {event_id} exceeded max age "
+                    f"({self.max_age_hours} hours, age={event_age}), "
+                    "marking as expired"
+                )
+                await self.storage.mark_outbox_expired(
+                    event_id,
+                    f"Exceeded max age ({self.max_age_hours} hours, age={event_age})",
+                )
+                return
+
+        # Check if max retries exceeded
+        if retry_count >= self.max_retries:
+            logger.warning(
+                f"Event {event_id} exceeded max retries ({self.max_retries}), "
+                "marking as permanently failed"
+            )
+            await self.storage.mark_outbox_permanently_failed(
+                event_id, f"Exceeded max retries ({self.max_retries})"
+            )
+            return
+
+        try:
+            # Build CloudEvent
+            ce = CloudEvent(
+                {
+                    "type": event["event_type"],
+                    "source": event["event_source"],
+                    "id": event_id,
+                },
+                event["event_data"],
+            )
+
+            # Convert to structured format
+            headers, body = to_structured(ce)
+
+            # Publish to Message Broker
+            if self._http_client is None:
+                raise RuntimeError("HTTP client not initialized")
+
+            response = await self._http_client.post(
+                self.broker_url,
+                headers=headers,
+                content=body,
+            )
+
+            # Check response
+            response.raise_for_status()
+
+            # Mark as published
+            await self.storage.mark_outbox_published(event_id)
+            logger.info(f"Successfully published event {event_id}")
+
+        except httpx.HTTPStatusError as e:
+            # HTTP error with status code - distinguish 4xx (client) vs 5xx (server)
+            status_code = e.response.status_code
+            error_msg = f"HTTP {status_code}: {str(e)}"
+
+            if 400 <= status_code < 500:
+                # Client error (4xx) - permanent failure, don't retry
+                logger.error(
+                    f"Permanent error for event {event_id}: {error_msg}. "
+                    "Marking as invalid (won't retry)"
+                )
+                await self.storage.mark_outbox_invalid(event_id, error_msg)
+            else:
+                # Server error (5xx) - temporary failure, retry
+                logger.warning(
+                    f"Server error for event {event_id} "
+                    f"(retry {retry_count + 1}/{self.max_retries}): {error_msg}"
+                )
+                await self.storage.mark_outbox_failed(event_id, error_msg)
+
+        except httpx.RequestError as e:
+            # Network error (connection timeout, DNS failure, etc.) - retry
+            error_msg = f"Network error: {str(e)}"
+            logger.warning(
+                f"Network error for event {event_id} "
+                f"(retry {retry_count + 1}/{self.max_retries}): {error_msg}"
+            )
+            await self.storage.mark_outbox_failed(event_id, error_msg)
+
+        except Exception as e:
+            # Unknown error - retry (safety net)
+            error_msg = f"{type(e).__name__}: {str(e)}"
+            logger.warning(
+                f"Unknown error for event {event_id} "
+                f"(retry {retry_count + 1}/{self.max_retries}): {error_msg}"
+            )
+            await self.storage.mark_outbox_failed(event_id, error_msg)

edda/outbox/transactional.py

@@ -0,0 +1,112 @@
+"""
+Transactional outbox helpers for reliable event publishing.
+
+This module provides utilities for sending events using the transactional
+outbox pattern. Events are written to the database within the same transaction
+as other business logic, ensuring atomicity.
+"""
+
+import logging
+import uuid
+from typing import TYPE_CHECKING, Any, cast
+
+from pydantic import BaseModel
+
+from edda.pydantic_utils import is_pydantic_instance, to_json_dict
+
+if TYPE_CHECKING:
+    from edda.context import WorkflowContext
+
+logger = logging.getLogger(__name__)
+
+
+async def send_event_transactional(
+    ctx: "WorkflowContext",
+    event_type: str,
+    event_source: str,
+    event_data: dict[str, Any] | BaseModel,
+    content_type: str = "application/json",
+) -> str:
+    """
+    Send an event using the transactional outbox pattern.
+
+    This function writes an event to the outbox table instead of sending it
+    directly. The event will be asynchronously published by the Outbox Relayer.
+
+    This ensures that event publishing is atomic with the workflow execution:
+    - If the workflow fails, the event is not published
+    - If the workflow succeeds, the event is guaranteed to be published
+
+    Example:
+        >>> # With dict
+        >>> @activity
+        ... async def reserve_inventory(ctx: WorkflowContext, order_id: str) -> dict:
+        ...     reservation_id = str(uuid.uuid4())
+        ...     await send_event_transactional(
+        ...         ctx,
+        ...         event_type="inventory.reserved",
+        ...         event_source="order-service",
+        ...         event_data={
+        ...             "order_id": order_id,
+        ...             "reservation_id": reservation_id,
+        ...         }
+        ...     )
+        ...     return {"reservation_id": reservation_id}
+        >>>
+        >>> # With Pydantic model (automatically converted to JSON)
+        >>> @activity
+        ... async def reserve_inventory_typed(ctx: WorkflowContext, order_id: str) -> dict:
+        ...     reservation_id = str(uuid.uuid4())
+        ...     event = InventoryReserved(
+        ...         order_id=order_id,
+        ...         reservation_id=reservation_id,
+        ...     )
+        ...     await send_event_transactional(
+        ...         ctx,
+        ...         event_type="inventory.reserved",
+        ...         event_source="order-service",
+        ...         event_data=event,
+        ...     )
+        ...     return {"reservation_id": reservation_id}
+
+    Args:
+        ctx: Workflow context
+        event_type: CloudEvent type (e.g., "order.created")
+        event_source: CloudEvent source (e.g., "order-service")
+        event_data: Event payload (JSON dict or Pydantic model)
+        content_type: Content type (defaults to "application/json")
+
+    Returns:
+        Event ID (UUID)
+
+    Raises:
+        Exception: If writing to outbox fails
+    """
+    # Check if in transaction
+    if not ctx.in_transaction():
+        logger.warning(
+            "send_event_transactional() called outside of a transaction. "
+            "Event will still be sent, but atomicity with other operations is not guaranteed. "
+            "Consider using @activity (with default transactional=True) or wrapping in ctx.transaction()."
+        )
+
+    # Convert Pydantic model to JSON dict
+    event_data_dict: dict[str, Any]
+    if is_pydantic_instance(event_data):
+        event_data_dict = to_json_dict(event_data)
+    else:
+        event_data_dict = cast(dict[str, Any], event_data)
+
+    # Generate event ID
+    event_id = str(uuid.uuid4())
+
+    # Write to outbox table
+    await ctx.storage.add_outbox_event(
+        event_id=event_id,
+        event_type=event_type,
+        event_source=event_source,
+        event_data=event_data_dict,
+        content_type=content_type,
+    )
+
+    return event_id

edda/pydantic_utils.py

@@ -0,0 +1,316 @@
+"""Pydantic integration utilities for Edda framework.
+
+This module provides utilities for detecting Pydantic models and converting
+between Pydantic models and JSON-compatible dictionaries.
+"""
+
+import types
+from enum import Enum
+from typing import Any, TypeVar, Union, cast, get_args, get_origin
+
+from pydantic import BaseModel
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def is_pydantic_model(obj: Any) -> bool:
+    """Check if an object is a Pydantic model class.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if obj is a Pydantic model class, False otherwise
+
+    Examples:
+        >>> from pydantic import BaseModel
+        >>> class User(BaseModel):
+        ...     name: str
+        >>> is_pydantic_model(User)
+        True
+        >>> is_pydantic_model(User(name="Alice"))
+        False
+        >>> is_pydantic_model(str)
+        False
+    """
+    try:
+        return isinstance(obj, type) and issubclass(obj, BaseModel)
+    except TypeError:
+        # issubclass raises TypeError for non-class objects
+        return False
+
+
+def is_pydantic_instance(obj: Any) -> bool:
+    """Check if an object is a Pydantic model instance.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if obj is a Pydantic model instance, False otherwise
+
+    Examples:
+        >>> from pydantic import BaseModel
+        >>> class User(BaseModel):
+        ...     name: str
+        >>> is_pydantic_instance(User(name="Alice"))
+        True
+        >>> is_pydantic_instance(User)
+        False
+        >>> is_pydantic_instance("Alice")
+        False
+    """
+    return isinstance(obj, BaseModel)
+
+
+def to_json_dict(obj: Any) -> Any:
+    """Convert a Pydantic model or Enum to a JSON-compatible value.
+
+    Recursively handles lists and dicts containing Pydantic models or Enums.
+
+    Args:
+        obj: Object to convert (Pydantic model, Enum, list, dict, or primitive)
+
+    Returns:
+        JSON-compatible value (dict for Pydantic, primitive for Enum, or original value)
+
+    Examples:
+        >>> from pydantic import BaseModel
+        >>> from enum import Enum
+        >>> class User(BaseModel):
+        ...     name: str
+        ...     age: int
+        >>> class Status(Enum):
+        ...     ACTIVE = "active"
+        >>> user = User(name="Alice", age=30)
+        >>> to_json_dict(user)
+        {'name': 'Alice', 'age': 30}
+        >>> to_json_dict(Status.ACTIVE)
+        'active'
+        >>> to_json_dict([user])
+        [{'name': 'Alice', 'age': 30}]
+        >>> to_json_dict({'name': 'Bob'})
+        {'name': 'Bob'}
+    """
+    if is_pydantic_instance(obj):
+        return obj.model_dump(mode="json")
+    if isinstance(obj, Enum):
+        return obj.value
+    if isinstance(obj, list):
+        return [to_json_dict(item) for item in obj]
+    if isinstance(obj, dict):
+        return {key: to_json_dict(value) for key, value in obj.items()}
+    return obj
+
+
+def from_json_dict(data: dict[str, Any], model: type[T]) -> T:
+    """Convert a JSON dictionary to a Pydantic model instance.
+
+    Args:
+        data: JSON-compatible dictionary
+        model: Pydantic model class
+
+    Returns:
+        Pydantic model instance
+
+    Raises:
+        ValidationError: If data does not match model schema
+
+    Examples:
+        >>> from pydantic import BaseModel
+        >>> class User(BaseModel):
+        ...     name: str
+        ...     age: int
+        >>> data = {'name': 'Alice', 'age': 30}
+        >>> user = from_json_dict(data, User)
+        >>> user.name
+        'Alice'
+        >>> user.age
+        30
+    """
+    return model.model_validate(data)
+
+
+def extract_pydantic_model_from_annotation(annotation: Any) -> type[BaseModel] | None:
+    """Extract Pydantic model class from a type annotation.
+
+    Handles various annotation patterns:
+    - Direct: User
+    - Optional: User | None, Optional[User]
+    - Generic: list[User], dict[str, User]
+
+    Args:
+        annotation: Type annotation to analyze
+
+    Returns:
+        Pydantic model class if found, None otherwise
+
+    Examples:
+        >>> from pydantic import BaseModel
+        >>> from typing import Optional
+        >>> class User(BaseModel):
+        ...     name: str
+        >>> extract_pydantic_model_from_annotation(User)
+        <class 'User'>
+        >>> extract_pydantic_model_from_annotation(User | None)
+        <class 'User'>
+        >>> extract_pydantic_model_from_annotation(Optional[User])
+        <class 'User'>
+        >>> extract_pydantic_model_from_annotation(str)
+        None
+    """
+    # Direct Pydantic model
+    if is_pydantic_model(annotation):
+        return cast(type[BaseModel], annotation)
+
+    # Handle Union types (e.g., User | None, Optional[User])
+    # Only check for Union types, not other generics (list, dict, etc.)
+    origin = get_origin(annotation)
+    # Check for typing.Union (Optional[X]) and types.UnionType (X | Y in Python 3.10+)
+    if origin is Union or origin is types.UnionType:
+        args = get_args(annotation)
+        for arg in args:
+            if is_pydantic_model(arg):
+                return cast(type[BaseModel], arg)
+
+    return None
+
+
+# ============================================================================
+# Enum utilities
+# ============================================================================
+
+
+def is_enum_class(obj: Any) -> bool:
+    """Check if an object is an Enum class.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if obj is an Enum class, False otherwise
+
+    Examples:
+        >>> from enum import Enum
+        >>> class Status(Enum):
+        ...     ACTIVE = "active"
+        >>> is_enum_class(Status)
+        True
+        >>> is_enum_class(Status.ACTIVE)
+        False
+        >>> is_enum_class(str)
+        False
+    """
+    try:
+        return isinstance(obj, type) and issubclass(obj, Enum)
+    except TypeError:
+        # issubclass raises TypeError for non-class objects
+        return False
+
+
+def is_enum_instance(obj: Any) -> bool:
+    """Check if an object is an Enum instance.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if obj is an Enum instance, False otherwise
+
+    Examples:
+        >>> from enum import Enum
+        >>> class Status(Enum):
+        ...     ACTIVE = "active"
+        >>> is_enum_instance(Status.ACTIVE)
+        True
+        >>> is_enum_instance(Status)
+        False
+        >>> is_enum_instance("active")
+        False
+    """
+    return isinstance(obj, Enum)
+
+
+def extract_enum_from_annotation(annotation: Any) -> type[Enum] | None:
+    """Extract Enum class from a type annotation.
+
+    Handles various annotation patterns:
+    - Direct: Status
+    - Optional: Status | None, Optional[Status]
+
+    Args:
+        annotation: Type annotation to analyze
+
+    Returns:
+        Enum class if found, None otherwise
+
+    Examples:
+        >>> from enum import Enum
+        >>> from typing import Optional
+        >>> class Status(Enum):
+        ...     ACTIVE = "active"
+        >>> extract_enum_from_annotation(Status)
+        <class 'Status'>
+        >>> extract_enum_from_annotation(Status | None)
+        <class 'Status'>
+        >>> extract_enum_from_annotation(Optional[Status])
+        <class 'Status'>
+        >>> extract_enum_from_annotation(str)
+        None
+    """
+    # Direct Enum class
+    if is_enum_class(annotation):
+        return cast(type[Enum], annotation)
+
+    # Handle Union types (e.g., Status | None, Optional[Status])
+    origin = get_origin(annotation)
+    # Check for typing.Union (Optional[X]) and types.UnionType (X | Y in Python 3.10+)
+    if origin is Union or origin is types.UnionType:
+        args = get_args(annotation)
+        for arg in args:
+            if is_enum_class(arg):
+                return cast(type[Enum], arg)
+
+    return None
+
+
+def enum_value_to_enum(value: Any, enum_class: type[Enum]) -> Enum:
+    """Convert a value (str or int) to Enum instance.
+
+    Args:
+        value: Value to convert (must match an Enum member's value)
+        enum_class: Enum class to convert to
+
+    Returns:
+        Enum instance
+
+    Raises:
+        ValueError: If value does not match any Enum member
+
+    Examples:
+        >>> from enum import Enum
+        >>> class Status(Enum):
+        ...     ACTIVE = "active"
+        ...     INACTIVE = "inactive"
+        >>> enum_value_to_enum("active", Status)
+        <Status.ACTIVE: 'active'>
+        >>> enum_value_to_enum("invalid", Status)
+        Traceback (most recent call last):
+        ...
+        ValueError: Cannot convert 'invalid' to Status
+    """
+    # Try by value first (most common case)
+    for member in enum_class:
+        if member.value == value:
+            return member
+
+    # Try by name (fallback for string values matching member names)
+    if isinstance(value, str):
+        # Try exact match first
+        if hasattr(enum_class, value):
+            return cast(Enum, getattr(enum_class, value))
+        # Try uppercase
+        if hasattr(enum_class, value.upper()):
+            return cast(Enum, getattr(enum_class, value.upper()))
+
+    raise ValueError(f"Cannot convert {value!r} to {enum_class.__name__}")