edda-framework 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

edda/outbox/relayer.py

@@ -48,6 +48,7 @@ class OutboxRelayer:
         max_retries: int = 3,
         batch_size: int = 10,
         max_age_hours: float | None = None,
+        wake_event: asyncio.Event | None = None,
     ):
         """
         Initialize the Outbox Relayer.
@@ -60,6 +61,8 @@ class OutboxRelayer:
             batch_size: Number of events to process per batch (default: 10)
             max_age_hours: Maximum event age in hours before expiration (default: None, disabled)
                           Events older than this are marked as 'expired' and won't be retried.
+            wake_event: Optional asyncio.Event to wake the relayer immediately when new
+                       events are added. Used with PostgreSQL LISTEN/NOTIFY integration.
         """
         self.storage = storage
         self.broker_url = broker_url
@@ -67,6 +70,7 @@ class OutboxRelayer:
         self.max_retries = max_retries
         self.batch_size = batch_size
         self.max_age_hours = max_age_hours
+        self._wake_event = wake_event
 
         self._task: asyncio.Task[Any] | None = None
         self._running = False
@@ -120,6 +124,8 @@ class OutboxRelayer:
         Main polling loop.
 
         Continuously polls the database for pending events and publishes them.
+        When wake_event is provided (PostgreSQL NOTIFY integration), wakes up
+        immediately on notification, otherwise falls back to poll_interval.
         """
         while self._running:
             try:
@@ -127,8 +133,21 @@ class OutboxRelayer:
             except Exception as e:
                 logger.error(f"Error in outbox relayer poll loop: {e}")
 
-            # Wait before next poll
-            await asyncio.sleep(self.poll_interval)
+            # Wait before next poll (with optional NOTIFY wake)
+            if self._wake_event is not None:
+                try:
+                    await asyncio.wait_for(
+                        self._wake_event.wait(),
+                        timeout=self.poll_interval,
+                    )
+                    # Clear the event for next notification
+                    self._wake_event.clear()
+                    logger.debug("Outbox relayer woken by NOTIFY")
+                except TimeoutError:
+                    # Fallback polling timeout reached
+                    pass
+            else:
+                await asyncio.sleep(self.poll_interval)
 
     async def _poll_and_publish(self) -> None:
         """

edda/storage/__init__.py

@@ -1,9 +1,17 @@
 """Storage layer for Edda framework."""
 
+from edda.storage.notify_base import (
+    NoopNotifyListener,
+    NotifyProtocol,
+    create_notify_listener,
+)
 from edda.storage.protocol import StorageProtocol
 from edda.storage.sqlalchemy_storage import SQLAlchemyStorage
 
 __all__ = [
     "StorageProtocol",
     "SQLAlchemyStorage",
+    "NotifyProtocol",
+    "NoopNotifyListener",
+    "create_notify_listener",
 ]

edda/storage/notify_base.py

@@ -0,0 +1,162 @@
+"""Base classes and protocols for notification systems.
+
+This module defines the NotifyProtocol interface and provides a NoopNotifyListener
+implementation for databases that don't support LISTEN/NOTIFY (SQLite, MySQL).
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Protocol, runtime_checkable
+
+logger = logging.getLogger(__name__)
+
+
+NotifyCallback = Callable[[str], Awaitable[None]]
+"""Type alias for notification callback functions.
+
+The callback receives the payload string from the notification.
+"""
+
+
+@runtime_checkable
+class NotifyProtocol(Protocol):
+    """Protocol for notification systems.
+
+    This protocol defines the interface for LISTEN/NOTIFY style notification
+    systems. Implementations should handle:
+    - Connection management with automatic reconnection
+    - Channel subscription/unsubscription
+    - Callback dispatch on notification receipt
+    """
+
+    async def start(self) -> None:
+        """Start the notification listener.
+
+        Establishes the connection and begins listening for notifications.
+        Should be called before any subscribe() calls.
+        """
+        ...
+
+    async def stop(self) -> None:
+        """Stop the notification listener.
+
+        Closes the connection and cleans up resources.
+        All subscriptions are automatically removed.
+        """
+        ...
+
+    async def subscribe(self, channel: str, callback: NotifyCallback) -> None:
+        """Subscribe to notifications on a channel.
+
+        Args:
+            channel: The channel name to listen on.
+            callback: Async function called when a notification arrives.
+                     Receives the payload string as its argument.
+        """
+        ...
+
+    async def unsubscribe(self, channel: str) -> None:
+        """Unsubscribe from notifications on a channel.
+
+        Args:
+            channel: The channel name to stop listening on.
+        """
+        ...
+
+    async def notify(self, channel: str, payload: str) -> None:
+        """Send a notification on a channel.
+
+        Args:
+            channel: The channel name to send the notification on.
+            payload: The payload string (typically JSON, max ~7500 bytes for PostgreSQL).
+        """
+        ...
+
+    @property
+    def is_connected(self) -> bool:
+        """Check if the listener is currently connected."""
+        ...
+
+
+class NoopNotifyListener:
+    """No-op implementation of NotifyProtocol for SQLite/MySQL.
+
+    This implementation does nothing - all methods are no-ops.
+    When using SQLite or MySQL, the application falls back to polling-based
+    updates with the default intervals (no reduction in polling frequency).
+    """
+
+    def __init__(self) -> None:
+        """Initialize the no-op listener."""
+        self._connected = False
+
+    async def start(self) -> None:
+        """No-op start - does nothing."""
+        self._connected = True
+        logger.debug("NoopNotifyListener started (no-op)")
+
+    async def stop(self) -> None:
+        """No-op stop - does nothing."""
+        self._connected = False
+        logger.debug("NoopNotifyListener stopped (no-op)")
+
+    async def subscribe(self, channel: str, _callback: NotifyCallback) -> None:
+        """No-op subscribe - callbacks will never be called.
+
+        Args:
+            channel: Ignored.
+            _callback: Ignored - will never be called.
+        """
+        logger.debug(f"NoopNotifyListener: subscribe to '{channel}' (no-op)")
+
+    async def unsubscribe(self, channel: str) -> None:
+        """No-op unsubscribe.
+
+        Args:
+            channel: Ignored.
+        """
+        logger.debug(f"NoopNotifyListener: unsubscribe from '{channel}' (no-op)")
+
+    async def notify(self, channel: str, payload: str) -> None:
+        """No-op notify - does nothing.
+
+        Args:
+            channel: Ignored.
+            payload: Ignored.
+        """
+        # Intentionally silent - this is called frequently during normal operation
+        pass
+
+    @property
+    def is_connected(self) -> bool:
+        """Always returns the internal connected state."""
+        return self._connected
+
+
+def create_notify_listener(db_url: str) -> NotifyProtocol:
+    """Create appropriate notify listener based on database type.
+
+    Args:
+        db_url: Database connection URL.
+
+    Returns:
+        PostgresNotifyListener for PostgreSQL, NoopNotifyListener for others.
+
+    Example:
+        >>> listener = create_notify_listener("postgresql://localhost/db")
+        >>> await listener.start()
+        >>> await listener.subscribe("my_channel", handle_notification)
+    """
+    if db_url.startswith("postgresql"):
+        # Import here to avoid requiring asyncpg when not using PostgreSQL
+        from edda.storage.pg_notify import PostgresNotifyListener
+
+        return PostgresNotifyListener(dsn=db_url)
+    else:
+        logger.info(
+            "Database URL does not start with 'postgresql', "
+            "using NoopNotifyListener (polling-based updates)"
+        )
+        return NoopNotifyListener()

edda/storage/pg_notify.py

@@ -0,0 +1,325 @@
+"""PostgreSQL LISTEN/NOTIFY implementation using asyncpg.
+
+This module provides a dedicated listener for PostgreSQL's LISTEN/NOTIFY
+mechanism, enabling near-instant notification delivery for workflow events.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from collections.abc import Awaitable, Callable
+from contextlib import suppress
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+NotifyCallback = Callable[[str], Awaitable[None]]
+
+
+class PostgresNotifyListener:
+    """PostgreSQL LISTEN/NOTIFY listener using asyncpg.
+
+    This class maintains a dedicated connection for LISTEN/NOTIFY operations.
+    It provides:
+    - Automatic reconnection on connection loss
+    - Channel subscription management
+    - Callback dispatch for notifications
+
+    Example:
+        >>> listener = PostgresNotifyListener(dsn="postgresql://localhost/db")
+        >>> await listener.start()
+        >>> await listener.subscribe("my_channel", handle_notification)
+        >>> # ... later
+        >>> await listener.stop()
+    """
+
+    def __init__(
+        self,
+        dsn: str,
+        reconnect_interval: float = 5.0,
+        max_reconnect_attempts: int | None = None,
+    ) -> None:
+        """Initialize the PostgreSQL notify listener.
+
+        Args:
+            dsn: PostgreSQL connection string (postgresql://user:pass@host/db).
+            reconnect_interval: Seconds to wait between reconnection attempts.
+            max_reconnect_attempts: Maximum number of reconnection attempts.
+                                   None means unlimited.
+        """
+        self._dsn = dsn
+        self._reconnect_interval = reconnect_interval
+        self._max_reconnect_attempts = max_reconnect_attempts
+
+        self._connection: Any = None  # asyncpg.Connection
+        self._callbacks: dict[str, list[NotifyCallback]] = {}
+        self._channel_handlers: dict[str, Callable[..., None]] = {}
+        self._running = False
+        self._reconnect_task: asyncio.Task[None] | None = None
+        self._lock = asyncio.Lock()
+
+    async def start(self) -> None:
+        """Start the notification listener.
+
+        Establishes the connection and begins listening for notifications.
+        Starts the automatic reconnection task.
+
+        Raises:
+            ImportError: If asyncpg is not installed.
+        """
+        if self._running:
+            logger.warning("PostgresNotifyListener already running")
+            return
+
+        self._running = True
+        await self._establish_connection()
+
+        # Start reconnection monitor
+        self._reconnect_task = asyncio.create_task(self._reconnect_loop())
+        logger.info("PostgresNotifyListener started")
+
+    async def stop(self) -> None:
+        """Stop the notification listener.
+
+        Closes the connection and stops the reconnection task.
+        """
+        self._running = False
+
+        # Cancel reconnection task
+        if self._reconnect_task is not None:
+            self._reconnect_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._reconnect_task
+            self._reconnect_task = None
+
+        # Close connection
+        await self._close_connection()
+        self._callbacks.clear()
+        logger.info("PostgresNotifyListener stopped")
+
+    async def subscribe(self, channel: str, callback: NotifyCallback) -> None:
+        """Subscribe to notifications on a channel.
+
+        Args:
+            channel: The PostgreSQL channel name to listen on.
+            callback: Async function called when a notification arrives.
+
+        Note:
+            Channel names must be valid PostgreSQL identifiers (max 63 chars).
+            Multiple callbacks can be registered for the same channel.
+        """
+        async with self._lock:
+            is_new_channel = channel not in self._callbacks
+
+            if channel not in self._callbacks:
+                self._callbacks[channel] = []
+            self._callbacks[channel].append(callback)
+
+            # Register listener if this is a new channel and we're connected
+            if is_new_channel and self._connection is not None:
+                try:
+                    await self._connection.add_listener(
+                        channel, self._create_notification_handler(channel)
+                    )
+                    logger.debug(f"Subscribed to channel: {channel}")
+                except Exception as e:
+                    logger.error(f"Failed to LISTEN on channel {channel}: {e}")
+
+    async def unsubscribe(self, channel: str) -> None:
+        """Unsubscribe from notifications on a channel.
+
+        Args:
+            channel: The PostgreSQL channel name to stop listening on.
+        """
+        async with self._lock:
+            if channel in self._callbacks:
+                del self._callbacks[channel]
+
+                # Remove listener if we're connected
+                if self._connection is not None:
+                    try:
+                        await self._connection.remove_listener(
+                            channel, self._create_notification_handler(channel)
+                        )
+                        logger.debug(f"Unsubscribed from channel: {channel}")
+                    except Exception as e:
+                        logger.error(f"Failed to UNLISTEN on channel {channel}: {e}")
+
+    async def notify(self, channel: str, payload: str) -> None:
+        """Send a notification on a channel.
+
+        Args:
+            channel: The PostgreSQL channel name.
+            payload: The payload string (max ~7500 bytes recommended).
+
+        Note:
+            This uses the existing connection pool from SQLAlchemy,
+            not the dedicated listener connection.
+        """
+        if self._connection is None:
+            logger.warning("Cannot send NOTIFY: not connected")
+            return
+
+        try:
+            # Use pg_notify function to properly escape the payload
+            await self._connection.execute("SELECT pg_notify($1, $2)", channel, payload)
+        except Exception as e:
+            logger.warning(f"Failed to send NOTIFY on channel {channel}: {e}")
+
+    @property
+    def is_connected(self) -> bool:
+        """Check if the listener is currently connected."""
+        return self._connection is not None and not self._connection.is_closed()
+
+    async def _establish_connection(self) -> None:
+        """Establish connection to PostgreSQL."""
+        try:
+            import asyncpg
+        except ImportError as e:
+            raise ImportError(
+                "asyncpg is required for PostgreSQL LISTEN/NOTIFY support. "
+                "Install it with: pip install edda[postgres-notify]"
+            ) from e
+
+        try:
+            self._connection = await asyncpg.connect(self._dsn)
+
+            # Re-subscribe to all channels (this also registers listeners)
+            await self._resubscribe_all()
+
+            logger.info("PostgresNotifyListener connected to database")
+        except Exception as e:
+            logger.error(f"Failed to connect to PostgreSQL: {e}")
+            self._connection = None
+            raise
+
+    async def _close_connection(self) -> None:
+        """Close the database connection."""
+        if self._connection is not None:
+            try:
+                await self._connection.close()
+            except Exception as e:
+                logger.warning(f"Error closing connection: {e}")
+            finally:
+                self._connection = None
+
+    async def _resubscribe_all(self) -> None:
+        """Re-subscribe to all registered channels after reconnection."""
+        if self._connection is None:
+            return
+
+        for channel in self._callbacks:
+            try:
+                # Register listener for this channel
+                await self._connection.add_listener(
+                    channel, self._create_notification_handler(channel)
+                )
+                logger.debug(f"Re-subscribed to channel: {channel}")
+            except Exception as e:
+                logger.error(f"Failed to re-subscribe to channel {channel}: {e}")
+
+    def _create_notification_handler(self, channel: str) -> Callable[..., None]:
+        """Create or retrieve a notification handler for a channel.
+
+        Args:
+            channel: The channel name.
+
+        Returns:
+            A handler function that can be passed to add_listener/remove_listener.
+        """
+        if channel not in self._channel_handlers:
+
+            def handler(_connection: Any, _pid: int, ch: str, payload: str) -> None:
+                """Handle incoming notification from PostgreSQL."""
+                callbacks = self._callbacks.get(ch, [])
+                for callback in callbacks:
+                    asyncio.create_task(self._safe_callback(callback, payload, ch))
+
+            self._channel_handlers[channel] = handler
+
+        return self._channel_handlers[channel]
+
+    async def _safe_callback(self, callback: NotifyCallback, payload: str, channel: str) -> None:
+        """Execute callback with error handling."""
+        try:
+            await callback(payload)
+        except Exception as e:
+            logger.error(
+                f"Error in notification callback for channel {channel}: {e}",
+                exc_info=True,
+            )
+
+    async def _reconnect_loop(self) -> None:
+        """Monitor connection and reconnect on failure."""
+        attempt = 0
+
+        with suppress(asyncio.CancelledError):
+            while self._running:
+                await asyncio.sleep(1)  # Check every second
+
+                if self._connection is None or self._connection.is_closed():
+                    attempt += 1
+                    if (
+                        self._max_reconnect_attempts is not None
+                        and attempt > self._max_reconnect_attempts
+                    ):
+                        logger.error(
+                            f"Max reconnection attempts ({self._max_reconnect_attempts}) "
+                            "exceeded, giving up"
+                        )
+                        break
+
+                    logger.info(
+                        f"Connection lost, attempting reconnection " f"(attempt {attempt})..."
+                    )
+
+                    try:
+                        await self._close_connection()
+                        await self._establish_connection()
+                        attempt = 0  # Reset on success
+                        logger.info("Reconnection successful")
+                    except Exception as e:
+                        logger.error(
+                            f"Reconnection failed: {e}, " f"retrying in {self._reconnect_interval}s"
+                        )
+                        await asyncio.sleep(self._reconnect_interval)
+
+
+def get_notify_channel_for_message(channel: str) -> str:
+    """Convert Edda channel name to PostgreSQL NOTIFY channel.
+
+    Uses a hash to ensure valid PostgreSQL identifier (max 63 chars).
+
+    Args:
+        channel: The Edda channel name.
+
+    Returns:
+        PostgreSQL-safe channel name.
+    """
+    import hashlib
+
+    h = hashlib.sha256(channel.encode()).hexdigest()[:16]
+    return f"edda_msg_{h}"
+
+
+def make_notify_payload(data: dict[str, Any]) -> str:
+    """Create JSON payload for NOTIFY.
+
+    Args:
+        data: Dictionary to serialize as JSON.
+
+    Returns:
+        JSON string (kept under 7500 bytes for PostgreSQL safety).
+    """
+    payload = json.dumps(data, separators=(",", ":"))  # Compact JSON
+    if len(payload) > 7500:
+        logger.warning(
+            f"NOTIFY payload exceeds recommended size " f"({len(payload)} > 7500 bytes), truncating"
+        )
+        # For safety, just include essential fields
+        minimal_data = {k: v for k, v in data.items() if k in ("wf_id", "ts")}
+        payload = json.dumps(minimal_data, separators=(",", ":"))
+    return payload

edda/storage/protocol.py

@@ -262,6 +262,7 @@ class StorageProtocol(Protocol):
         instance_id_filter: str | None = None,
         started_after: datetime | None = None,
         started_before: datetime | None = None,
+        input_filters: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
         """
         List workflow instances with cursor-based pagination and filtering.
@@ -277,6 +278,9 @@ class StorageProtocol(Protocol):
             instance_id_filter: Optional instance ID filter (partial match, case-insensitive)
             started_after: Filter instances started after this datetime (inclusive)
             started_before: Filter instances started before this datetime (inclusive)
+            input_filters: Filter by input data values. Keys are JSON paths
+                (e.g., "order_id" or "customer.email"), values are expected
+                values (exact match). All filters are AND-combined.
 
         Returns:
             Dictionary containing:
@@ -860,7 +864,7 @@ class StorageProtocol(Protocol):
     # Workflow Resumption Methods
     # -------------------------------------------------------------------------
 
-    async def find_resumable_workflows(self) -> list[dict[str, Any]]:
+    async def find_resumable_workflows(self, limit: int | None = None) -> list[dict[str, Any]]:
         """
         Find workflows that are ready to be resumed.
 
@@ -873,6 +877,10 @@ class StorageProtocol(Protocol):
         This allows immediate resumption after message delivery rather than
         waiting for the stale lock cleanup cycle (60+ seconds).
 
+        Args:
+            limit: Optional maximum number of workflows to return.
+                   If None, returns all resumable workflows.
+
         Returns:
             List of resumable workflows.
             Each item contains: instance_id, workflow_name