edda-framework 0.1.0__py3-none-any.whl

edda/hooks.py

@@ -0,0 +1,284 @@
+"""
+Hook system for extending Edda with custom observability and monitoring.
+
+This module provides a Protocol-based hook system that allows users to integrate
+their own observability tools (Logfire, Datadog, Jaeger, etc.) without coupling
+the framework to any specific tool.
+
+Example:
+    >>> from edda.hooks import WorkflowHooks
+    >>>
+    >>> class MyHooks(WorkflowHooks):
+    ...     async def on_workflow_start(self, instance_id, workflow_name, input_data):
+    ...         print(f"Workflow {workflow_name} started: {instance_id}")
+    ...
+    ...     async def on_activity_complete(self, instance_id, step, activity_name, result, cache_hit):
+    ...         print(f"Activity {activity_name} completed (cache_hit={cache_hit})")
+    >>>
+    >>> app = EddaApp(service_name="my-service", db_url="...", hooks=MyHooks())
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from typing import Any, Protocol
+
+
+class WorkflowHooks(Protocol):
+    """
+    Protocol for workflow lifecycle hooks.
+
+    Users can implement this protocol to add custom observability, logging,
+    or monitoring to their workflows. All methods are optional - implement
+    only the ones you need.
+
+    The framework will check if a hook method exists before calling it, so
+    partial implementations are fully supported.
+    """
+
+    async def on_workflow_start(
+        self, instance_id: str, workflow_name: str, input_data: dict[str, Any]
+    ) -> None:
+        """
+        Called when a workflow starts execution.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            workflow_name: Name of the workflow function
+            input_data: Input parameters passed to the workflow
+        """
+        ...
+
+    async def on_workflow_complete(self, instance_id: str, workflow_name: str, result: Any) -> None:
+        """
+        Called when a workflow completes successfully.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            workflow_name: Name of the workflow function
+            result: Return value from the workflow
+        """
+        ...
+
+    async def on_workflow_failed(
+        self, instance_id: str, workflow_name: str, error: Exception
+    ) -> None:
+        """
+        Called when a workflow fails with an exception.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            workflow_name: Name of the workflow function
+            error: Exception that caused the failure
+        """
+        ...
+
+    async def on_workflow_cancelled(self, instance_id: str, workflow_name: str) -> None:
+        """
+        Called when a workflow is cancelled.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            workflow_name: Name of the workflow function
+        """
+        ...
+
+    async def on_activity_start(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        is_replaying: bool,
+    ) -> None:
+        """
+        Called before an activity executes.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            step: Step number in the workflow
+            activity_name: Name of the activity function
+            is_replaying: True if this is a replay (cached result)
+        """
+        ...
+
+    async def on_activity_complete(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        result: Any,
+        cache_hit: bool,
+    ) -> None:
+        """
+        Called after an activity completes successfully.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            step: Step number in the workflow
+            activity_name: Name of the activity function
+            result: Return value from the activity
+            cache_hit: True if result was retrieved from cache (replay)
+        """
+        ...
+
+    async def on_activity_failed(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        error: Exception,
+    ) -> None:
+        """
+        Called when an activity fails with an exception.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            step: Step number in the workflow
+            activity_name: Name of the activity function
+            error: Exception that caused the failure
+        """
+        ...
+
+    async def on_activity_retry(
+        self,
+        instance_id: str,
+        activity_id: str,
+        activity_name: str,
+        error: Exception,
+        attempt: int,
+        delay: float,
+    ) -> None:
+        """
+        Called when an activity is about to be retried after a failure.
+
+        This hook is called BEFORE the retry delay (asyncio.sleep), allowing
+        observability tools to track retry attempts in real-time.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            activity_id: Activity ID (e.g., "my_activity:1")
+            activity_name: Name of the activity function
+            error: Exception that caused the failure
+            attempt: Current attempt number (1-indexed, before retry)
+            delay: Backoff delay in seconds before the next retry
+        """
+        ...
+
+    async def on_event_sent(
+        self,
+        event_type: str,
+        event_source: str,
+        event_data: dict[str, Any],
+    ) -> None:
+        """
+        Called when an event is sent (transactional outbox).
+
+        Args:
+            event_type: CloudEvents type
+            event_source: CloudEvents source
+            event_data: Event payload
+        """
+        ...
+
+    async def on_event_received(
+        self,
+        instance_id: str,
+        event_type: str,
+        event_data: dict[str, Any],
+    ) -> None:
+        """
+        Called when a workflow receives an awaited event.
+
+        Args:
+            instance_id: Unique workflow instance ID
+            event_type: CloudEvents type
+            event_data: Event payload
+        """
+        ...
+
+
+# Base class for convenient partial implementations
+class HooksBase(WorkflowHooks, ABC):
+    """
+    Abstract base class for WorkflowHooks implementations.
+
+    This can be used as a base class for partial implementations,
+    so you don't have to implement all methods.
+
+    Example:
+        >>> class MyHooks(HooksBase):
+        ...     async def on_workflow_start(self, instance_id, workflow_name, input_data):
+        ...         print(f"Workflow started: {workflow_name}")
+        ...     # Other methods are no-ops (inherited from HooksBase)
+    """
+
+    async def on_workflow_start(
+        self, instance_id: str, workflow_name: str, input_data: dict[str, Any]
+    ) -> None:
+        pass
+
+    async def on_workflow_complete(self, instance_id: str, workflow_name: str, result: Any) -> None:
+        pass
+
+    async def on_workflow_failed(
+        self, instance_id: str, workflow_name: str, error: Exception
+    ) -> None:
+        pass
+
+    async def on_workflow_cancelled(self, instance_id: str, workflow_name: str) -> None:
+        pass
+
+    async def on_activity_start(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        is_replaying: bool,
+    ) -> None:
+        pass
+
+    async def on_activity_complete(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        result: Any,
+        cache_hit: bool,
+    ) -> None:
+        pass
+
+    async def on_activity_failed(
+        self,
+        instance_id: str,
+        step: int,
+        activity_name: str,
+        error: Exception,
+    ) -> None:
+        pass
+
+    async def on_activity_retry(
+        self,
+        instance_id: str,
+        activity_id: str,
+        activity_name: str,
+        error: Exception,
+        attempt: int,
+        delay: float,
+    ) -> None:
+        pass
+
+    async def on_event_sent(
+        self,
+        event_type: str,
+        event_source: str,
+        event_data: dict[str, Any],
+    ) -> None:
+        pass
+
+    async def on_event_received(
+        self,
+        instance_id: str,
+        event_type: str,
+        event_data: dict[str, Any],
+    ) -> None:
+        pass

edda/locking.py

@@ -0,0 +1,322 @@
+"""
+Distributed locking utilities for Edda framework.
+
+This module provides helper functions and context managers for working with
+distributed locks in multi-pod deployments.
+"""
+
+import asyncio
+import os
+import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager, suppress
+from typing import Any
+
+from edda.storage.protocol import StorageProtocol
+
+
+def generate_worker_id(service_name: str) -> str:
+    """
+    Generate a unique worker ID for this process.
+
+    The worker ID combines the service name, process ID, and a random UUID
+    to ensure uniqueness across pods and restarts.
+
+    Args:
+        service_name: Name of the service (e.g., "order-service")
+
+    Returns:
+        Unique worker ID (e.g., "order-service-12345-a1b2c3d4")
+    """
+    pid = os.getpid()
+    unique_id = uuid.uuid4().hex[:8]
+    return f"{service_name}-{pid}-{unique_id}"
+
+
+async def acquire_lock_with_retry(
+    storage: StorageProtocol,
+    instance_id: str,
+    worker_id: str,
+    max_retries: int = 3,
+    retry_delay: float = 0.1,
+    timeout_seconds: int = 30,
+) -> bool:
+    """
+    Try to acquire lock with retries.
+
+    This is useful in high-contention scenarios where multiple workers
+    are trying to acquire the same lock simultaneously.
+
+    Args:
+        storage: Storage backend
+        instance_id: Workflow instance to lock
+        worker_id: Unique worker identifier
+        max_retries: Maximum number of retry attempts
+        retry_delay: Delay between retries in seconds
+        timeout_seconds: Lock timeout in seconds
+
+    Returns:
+        True if lock was acquired, False otherwise
+    """
+    for attempt in range(max_retries):
+        if await storage.try_acquire_lock(instance_id, worker_id, timeout_seconds):
+            return True
+
+        if attempt < max_retries - 1:
+            await asyncio.sleep(retry_delay)
+
+    return False
+
+
+async def ensure_lock_held(
+    storage: StorageProtocol,
+    instance_id: str,
+    worker_id: str,
+) -> None:
+    """
+    Verify that we still hold the lock for a workflow instance.
+
+    Raises RuntimeError if the lock is not held by this worker.
+
+    Args:
+        storage: Storage backend
+        instance_id: Workflow instance
+        worker_id: Unique worker identifier
+
+    Raises:
+        RuntimeError: If lock is not held by this worker
+    """
+    instance = await storage.get_instance(instance_id)
+    if instance is None:
+        raise RuntimeError(f"Workflow instance {instance_id} not found")
+
+    if instance.get("locked_by") != worker_id:
+        raise RuntimeError(
+            f"Lock lost for instance {instance_id}. "
+            f"Current lock holder: {instance.get('locked_by')}"
+        )
+
+
+@asynccontextmanager
+async def workflow_lock(
+    storage: StorageProtocol,
+    instance_id: str,
+    worker_id: str,
+    timeout_seconds: int = 300,
+    refresh_interval: float | None = None,
+) -> AsyncIterator[None]:
+    """
+    Context manager for acquiring and releasing workflow locks.
+
+    Automatically refreshes the lock periodically if refresh_interval is provided.
+
+    Example:
+        >>> async with workflow_lock(storage, instance_id, worker_id):
+        ...     # Execute workflow
+        ...     pass
+
+    Args:
+        storage: Storage backend
+        instance_id: Workflow instance to lock
+        worker_id: Unique worker identifier
+        timeout_seconds: Lock timeout in seconds
+        refresh_interval: Optional interval for lock refresh (seconds)
+
+    Yields:
+        None (lock is held during context)
+
+    Raises:
+        RuntimeError: If lock cannot be acquired
+    """
+    # Try to acquire lock
+    acquired = await storage.try_acquire_lock(instance_id, worker_id, timeout_seconds)
+    if not acquired:
+        raise RuntimeError(f"Failed to acquire lock for instance {instance_id}")
+
+    refresh_task: asyncio.Task[Any] | None = None
+
+    try:
+        # Start lock refresh task if requested
+        if refresh_interval is not None:
+            refresh_task = asyncio.create_task(
+                _refresh_lock_periodically(storage, instance_id, worker_id, refresh_interval)
+            )
+
+        yield
+
+    finally:
+        # Cancel refresh task
+        if refresh_task is not None:
+            refresh_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await refresh_task
+
+        # Release lock
+        await storage.release_lock(instance_id, worker_id)
+
+
+async def _refresh_lock_periodically(
+    storage: StorageProtocol,
+    instance_id: str,
+    worker_id: str,
+    interval: float,
+) -> None:
+    """
+    Periodically refresh a lock to prevent timeout.
+
+    This is a background task that runs while a workflow is executing.
+
+    Args:
+        storage: Storage backend
+        instance_id: Workflow instance
+        worker_id: Unique worker identifier
+        interval: Refresh interval in seconds
+    """
+    with suppress(asyncio.CancelledError):
+        while True:
+            await asyncio.sleep(interval)
+
+            # Refresh the lock
+            success = await storage.refresh_lock(instance_id, worker_id)
+            if not success:
+                # Lock was lost - this shouldn't happen in normal operation
+                raise RuntimeError(
+                    f"Lost lock for instance {instance_id} during refresh. "
+                    "Workflow execution may be compromised."
+                )
+
+
+async def cleanup_stale_locks_periodically(
+    storage: StorageProtocol,
+    interval: int = 60,
+) -> None:
+    """
+    Background task to periodically clean up stale locks.
+
+    This should be run as a background task in your application to ensure
+    that locks from crashed workers don't block workflows indefinitely.
+
+    Note: This function only cleans up locks without resuming workflows.
+    For automatic workflow resumption, use auto_resume_stale_workflows_periodically().
+
+    Example:
+        >>> asyncio.create_task(
+        ...     cleanup_stale_locks_periodically(storage, interval=60)
+        ... )
+
+    Args:
+        storage: Storage backend
+        interval: Cleanup interval in seconds (default: 60)
+    """
+    with suppress(asyncio.CancelledError):
+        while True:
+            await asyncio.sleep(interval)
+
+            # Clean up stale locks
+            workflows = await storage.cleanup_stale_locks()
+
+            if len(workflows) > 0:
+                # Log cleanup (in a real implementation, use proper logging)
+                print(f"Cleaned up {len(workflows)} stale locks")
+
+
+async def auto_resume_stale_workflows_periodically(
+    storage: StorageProtocol,
+    replay_engine: Any,
+    interval: int = 60,
+) -> None:
+    """
+    Background task to periodically clean up stale locks and auto-resume workflows.
+
+    This combines lock cleanup with automatic workflow resumption, ensuring
+    that workflows interrupted by worker crashes are automatically recovered.
+
+    Example:
+        >>> asyncio.create_task(
+        ...     auto_resume_stale_workflows_periodically(
+        ...         storage, replay_engine, interval=60
+        ...     )
+        ... )
+
+    Args:
+        storage: Storage backend
+        replay_engine: ReplayEngine instance for resuming workflows
+        interval: Cleanup interval in seconds (default: 60)
+    """
+    with suppress(asyncio.CancelledError):
+        while True:
+            await asyncio.sleep(interval)
+
+            # Clean up stale locks and get workflows to resume
+            workflows_to_resume = await storage.cleanup_stale_locks()
+
+            if len(workflows_to_resume) > 0:
+                # Log cleanup (in a real implementation, use proper logging)
+                print(f"Cleaned up {len(workflows_to_resume)} stale locks")
+
+                # Auto-resume workflows
+                for workflow in workflows_to_resume:
+                    instance_id = workflow["instance_id"]
+                    workflow_name = workflow["workflow_name"]
+                    source_hash = workflow["source_hash"]
+                    status = workflow.get("status", "running")
+
+                    try:
+                        # Special handling for workflows in compensating state
+                        if status == "compensating":
+                            # Workflow crashed during compensation execution
+                            # Only re-execute compensations, don't run workflow function
+                            print(
+                                f"Auto-resuming compensating workflow: {instance_id} "
+                                f"(compensation recovery only, no workflow execution)"
+                            )
+                            success = await replay_engine.resume_compensating_workflow(instance_id)
+                            if success:
+                                print(f"Successfully completed compensations for: {instance_id}")
+                            else:
+                                print(f"Failed to complete compensations for: {instance_id}")
+                            continue
+
+                        # Normal workflow resumption (status='running')
+                        # Check if workflow definition matches current Saga registry
+                        # This prevents resuming workflows with outdated/incompatible code
+                        current_definition = await storage.get_current_workflow_definition(
+                            workflow_name
+                        )
+
+                        if current_definition is None:
+                            print(
+                                f"Skipping auto-resume for {instance_id}: "
+                                f"workflow '{workflow_name}' not found in registry"
+                            )
+                            continue
+
+                        if current_definition["source_hash"] != source_hash:
+                            print(
+                                f"Skipping auto-resume for {instance_id}: "
+                                f"workflow definition has changed "
+                                f"(old hash: {source_hash[:8]}..., "
+                                f"new hash: {current_definition['source_hash'][:8]}...)"
+                            )
+                            continue
+
+                        # Hash matches - safe to resume
+                        print(f"Auto-resuming workflow: {workflow_name} (instance: {instance_id})")
+                        await replay_engine.resume_by_name(instance_id, workflow_name)
+                        print(f"Successfully resumed workflow: {instance_id}")
+                    except Exception as e:
+                        # Log error but continue with other workflows
+                        # In a real implementation, use proper logging
+                        print(f"Failed to auto-resume workflow {instance_id}: {e}")
+
+
+class LockNotAcquiredError(Exception):
+    """Raised when a lock cannot be acquired."""
+
+    pass
+
+
+class LockLostError(Exception):
+    """Raised when a lock is unexpectedly lost during execution."""
+
+    pass

edda/outbox/__init__.py

@@ -0,0 +1,15 @@
+"""
+Transactional Outbox Pattern Implementation.
+
+This module provides reliable event publishing using the transactional outbox pattern.
+Events are first written to the database, then asynchronously published by a background
+relayer process.
+"""
+
+from edda.outbox.relayer import OutboxRelayer
+from edda.outbox.transactional import send_event_transactional
+
+__all__ = [
+    "OutboxRelayer",
+    "send_event_transactional",
+]