pyworkflow-engine 0.1.7__py3-none-any.whl

pyworkflow/primitives/sleep.py

@@ -0,0 +1,100 @@
+"""
+Sleep primitive for workflow delays.
+
+Allows workflows to pause execution for a specified duration without
+holding resources. The workflow will suspend and can be resumed after
+the delay period.
+"""
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+from loguru import logger
+
+from pyworkflow.context import get_context, has_context
+from pyworkflow.utils.duration import parse_duration
+
+
+async def sleep(
+    duration: str | int | float | timedelta | datetime,
+    name: str | None = None,
+) -> None:
+    """
+    Suspend workflow execution for a specified duration.
+
+    Different contexts handle sleep differently:
+    - MockContext: Skips sleep (configurable)
+    - LocalContext: Durable sleep with event sourcing
+    - AWSContext: AWS native wait (no compute charges)
+
+    If called outside a workflow context, falls back to asyncio.sleep.
+
+    Args:
+        duration: How long to sleep:
+            - str: Duration string ("5s", "2m", "1h", "3d", "1w")
+            - int/float: Seconds
+            - timedelta: Time duration
+            - datetime: Sleep until this specific time
+        name: Optional name for this sleep (for debugging)
+
+    Examples:
+        # Sleep for 30 seconds
+        await sleep("30s")
+
+        # Sleep for 5 minutes
+        await sleep("5m")
+        await sleep(300)  # Same as above
+
+        # Sleep for 1 hour
+        await sleep("1h")
+        await sleep(timedelta(hours=1))
+
+        # Named sleep for debugging
+        await sleep("5m", name="wait_for_rate_limit")
+    """
+    # Check for workflow context
+    if has_context():
+        ctx = get_context()
+        duration_seconds = _calculate_delay_seconds(duration)
+
+        logger.debug(
+            f"Sleep {duration_seconds}s via {ctx.__class__.__name__}",
+            run_id=ctx.run_id,
+            workflow_name=ctx.workflow_name,
+        )
+
+        await ctx.sleep(duration_seconds)
+        return
+
+    # No context available - use regular asyncio.sleep
+    duration_seconds = _calculate_delay_seconds(duration)
+    logger.debug(
+        f"Sleep called outside workflow context, using asyncio.sleep for {duration_seconds}s"
+    )
+    await asyncio.sleep(duration_seconds)
+
+
+def _calculate_resume_time(duration: str | int | float | timedelta | datetime) -> datetime:
+    """Calculate when the sleep should resume."""
+    if isinstance(duration, datetime):
+        return duration
+
+    delay_seconds = _calculate_delay_seconds(duration)
+    return datetime.now(UTC) + timedelta(seconds=delay_seconds)
+
+
+def _calculate_delay_seconds(duration: str | int | float | timedelta | datetime) -> int:
+    """Calculate delay in seconds."""
+    if isinstance(duration, datetime):
+        now = datetime.now(UTC)
+        if duration <= now:
+            raise ValueError(f"Cannot sleep until past time: {duration} (now: {now})")
+        delta = duration - now
+        return int(delta.total_seconds())
+
+    if isinstance(duration, timedelta):
+        return int(duration.total_seconds())
+    elif isinstance(duration, str):
+        return parse_duration(duration)
+    else:
+        return int(duration)

pyworkflow/runtime/__init__.py

@@ -0,0 +1,21 @@
+"""
+PyWorkflow Runtime Abstraction Layer.
+
+Runtimes determine WHERE workflow code executes:
+- LocalRuntime: In-process execution (for CI, testing, simple scripts)
+- CeleryRuntime: Distributed execution via Celery workers
+- LambdaRuntime: AWS Lambda execution (future)
+- DurableLambdaRuntime: AWS Durable Lambda execution (future)
+"""
+
+from pyworkflow.runtime.base import Runtime
+from pyworkflow.runtime.factory import get_runtime, register_runtime, validate_runtime_durable
+from pyworkflow.runtime.local import LocalRuntime
+
+__all__ = [
+    "Runtime",
+    "LocalRuntime",
+    "get_runtime",
+    "register_runtime",
+    "validate_runtime_durable",
+]

pyworkflow/runtime/base.py

@@ -0,0 +1,179 @@
+"""
+Abstract base class for workflow execution runtimes.
+
+Runtimes are responsible for:
+- Starting workflow executions
+- Resuming suspended workflows
+- Scheduling wake-up times for sleeps
+"""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from pyworkflow.storage.base import StorageBackend
+
+
+class Runtime(ABC):
+    """
+    Abstract base class for workflow execution runtimes.
+
+    A runtime determines WHERE and HOW workflow code executes.
+    Different runtimes support different capabilities (durable vs transient).
+    """
+
+    @abstractmethod
+    async def start_workflow(
+        self,
+        workflow_func: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        run_id: str,
+        workflow_name: str,
+        storage: Optional["StorageBackend"],
+        durable: bool,
+        idempotency_key: str | None = None,
+        max_duration: str | None = None,
+        metadata: dict | None = None,
+    ) -> str:
+        """
+        Start a new workflow execution.
+
+        Args:
+            workflow_func: The workflow function to execute
+            args: Positional arguments for the workflow
+            kwargs: Keyword arguments for the workflow
+            run_id: Unique identifier for this run
+            workflow_name: Name of the workflow
+            storage: Storage backend (None for transient workflows)
+            durable: Whether this is a durable workflow
+            idempotency_key: Optional key for idempotent execution
+            max_duration: Optional maximum duration for the workflow
+            metadata: Optional metadata dictionary
+
+        Returns:
+            The run_id of the started workflow
+        """
+        pass
+
+    @abstractmethod
+    async def resume_workflow(
+        self,
+        run_id: str,
+        storage: "StorageBackend",
+    ) -> Any:
+        """
+        Resume a suspended workflow.
+
+        Args:
+            run_id: The run_id of the workflow to resume
+            storage: Storage backend containing workflow state
+
+        Returns:
+            The result of the workflow execution
+        """
+        pass
+
+    @abstractmethod
+    async def schedule_wake(
+        self,
+        run_id: str,
+        wake_time: datetime,
+        storage: "StorageBackend",
+    ) -> None:
+        """
+        Schedule a workflow to be resumed at a specific time.
+
+        Args:
+            run_id: The run_id of the workflow to wake
+            wake_time: When to resume the workflow
+            storage: Storage backend
+        """
+        pass
+
+    async def schedule_resume(
+        self,
+        run_id: str,
+        storage: "StorageBackend",
+    ) -> None:
+        """
+        Schedule a workflow to be resumed immediately.
+
+        This is called by resume_hook() after recording the hook event.
+        Each runtime implements this differently:
+        - CeleryRuntime: Schedules an async Celery task
+        - LocalRuntime: Calls resume_workflow directly (in-process)
+
+        Args:
+            run_id: The run_id of the workflow to resume
+            storage: Storage backend
+        """
+        # Default implementation: no-op
+        # Subclasses override if they support async scheduling
+        pass
+
+    @abstractmethod
+    async def start_child_workflow(
+        self,
+        workflow_func: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        child_run_id: str,
+        workflow_name: str,
+        storage: "StorageBackend",
+        parent_run_id: str,
+        child_id: str,
+        wait_for_completion: bool,
+    ) -> None:
+        """
+        Start a child workflow execution (fire-and-forget).
+
+        Child workflows run in the background and notify the parent
+        when completed/failed. If wait_for_completion=True, the parent
+        will be resumed when the child finishes.
+
+        Args:
+            workflow_func: The child workflow function to execute
+            args: Positional arguments for the child workflow
+            kwargs: Keyword arguments for the child workflow
+            child_run_id: Unique identifier for the child run
+            workflow_name: Name of the child workflow
+            storage: Storage backend
+            parent_run_id: Run ID of the parent workflow
+            child_id: Deterministic child ID for replay
+            wait_for_completion: Whether parent is waiting for child to complete
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Runtime identifier.
+
+        Returns:
+            String identifier for this runtime (e.g., "local", "celery")
+        """
+        pass
+
+    @property
+    def supports_durable(self) -> bool:
+        """
+        Whether this runtime supports durable (event-sourced) workflows.
+
+        Returns:
+            True if durable workflows are supported
+        """
+        return True
+
+    @property
+    def supports_transient(self) -> bool:
+        """
+        Whether this runtime supports transient (non-durable) workflows.
+
+        Returns:
+            True if transient workflows are supported
+        """
+        return True

pyworkflow/runtime/celery.py

@@ -0,0 +1,310 @@
+"""
+Celery runtime - executes workflows on distributed Celery workers.
+
+The Celery runtime is ideal for:
+- Production deployments
+- Distributed execution across multiple workers
+- Long-running workflows with sleeps and webhooks
+- High availability and scalability
+"""
+
+import os
+from collections.abc import Callable
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Optional
+
+from loguru import logger
+
+from pyworkflow.runtime.base import Runtime
+
+if TYPE_CHECKING:
+    from pyworkflow.storage.base import StorageBackend
+
+
+class CeleryRuntime(Runtime):
+    """
+    Execute workflows on distributed Celery workers.
+
+    This runtime dispatches workflow execution to Celery workers,
+    enabling distributed processing and automatic resumption of
+    suspended workflows.
+
+    Note: This runtime only supports durable workflows since
+    Celery execution requires state persistence for proper
+    task routing and resumption.
+    """
+
+    def __init__(
+        self,
+        broker_url: str | None = None,
+        result_backend: str | None = None,
+    ):
+        """
+        Initialize Celery runtime.
+
+        Args:
+            broker_url: Celery broker URL (default: from env or redis://localhost:6379/0)
+            result_backend: Result backend URL (default: from env or redis://localhost:6379/1)
+        """
+        self._broker_url: str = (
+            broker_url
+            or os.getenv("PYWORKFLOW_CELERY_BROKER", "redis://localhost:6379/0")
+            or "redis://localhost:6379/0"
+        )
+        self._result_backend: str = (
+            result_backend
+            or os.getenv("PYWORKFLOW_CELERY_RESULT_BACKEND", "redis://localhost:6379/1")
+            or "redis://localhost:6379/1"
+        )
+
+    @property
+    def name(self) -> str:
+        return "celery"
+
+    @property
+    def supports_durable(self) -> bool:
+        return True
+
+    @property
+    def supports_transient(self) -> bool:
+        # Celery runtime requires durable workflows for proper state management
+        return False
+
+    @property
+    def broker_url(self) -> str:
+        """Get the configured broker URL."""
+        return self._broker_url
+
+    @property
+    def result_backend(self) -> str:
+        """Get the configured result backend URL."""
+        return self._result_backend
+
+    def _get_storage_config(self, storage: Optional["StorageBackend"]) -> dict | None:
+        """
+        Convert storage backend to configuration dict for Celery tasks.
+
+        Args:
+            storage: Storage backend instance
+
+        Returns:
+            Configuration dict or None
+        """
+        from pyworkflow.storage.config import storage_to_config
+
+        config = storage_to_config(storage)
+
+        # In-memory storage cannot be shared across Celery workers
+        if config and config.get("type") == "memory":
+            logger.warning(
+                "InMemoryStorageBackend cannot be used with Celery runtime. "
+                "Falling back to FileStorageBackend."
+            )
+            return {"type": "file"}
+
+        return config
+
+    async def start_workflow(
+        self,
+        workflow_func: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        run_id: str,
+        workflow_name: str,
+        storage: Optional["StorageBackend"],
+        durable: bool,
+        idempotency_key: str | None = None,
+        max_duration: str | None = None,
+        metadata: dict | None = None,
+    ) -> str:
+        """
+        Start a workflow execution by dispatching to Celery workers.
+
+        The workflow will be queued and executed by an available worker.
+        """
+        from pyworkflow.celery.tasks import start_workflow_task
+        from pyworkflow.serialization.encoder import serialize_args, serialize_kwargs
+
+        if not durable:
+            raise ValueError(
+                "Celery runtime requires durable=True. "
+                "Use the 'local' runtime for transient workflows."
+            )
+
+        logger.info(
+            f"Dispatching workflow to Celery: {workflow_name}",
+            run_id=run_id,
+            workflow_name=workflow_name,
+        )
+
+        # Serialize arguments for Celery transport
+        args_json = serialize_args(*args)
+        kwargs_json = serialize_kwargs(**kwargs)
+
+        # Get storage configuration for workers
+        storage_config = self._get_storage_config(storage)
+
+        # Dispatch to Celery worker
+        task_result = start_workflow_task.delay(
+            workflow_name=workflow_name,
+            args_json=args_json,
+            kwargs_json=kwargs_json,
+            run_id=run_id,
+            storage_config=storage_config,
+            idempotency_key=idempotency_key,
+        )
+
+        logger.info(
+            f"Workflow dispatched to Celery: {workflow_name}",
+            run_id=run_id,
+            task_id=task_result.id,
+        )
+
+        # Return the run_id (the actual run_id is generated by the worker)
+        # For now, we return a pending status indicator
+        # The actual run_id can be obtained from the task result
+        return run_id
+
+    async def resume_workflow(
+        self,
+        run_id: str,
+        storage: "StorageBackend",
+    ) -> Any:
+        """
+        Resume a suspended workflow by dispatching to Celery workers.
+        """
+        from pyworkflow.celery.tasks import resume_workflow_task
+
+        logger.info(
+            f"Dispatching workflow resume to Celery: {run_id}",
+            run_id=run_id,
+        )
+
+        # Get storage configuration for workers
+        storage_config = self._get_storage_config(storage)
+
+        # Dispatch to Celery worker
+        task_result = resume_workflow_task.delay(
+            run_id=run_id,
+            storage_config=storage_config,
+        )
+
+        logger.info(
+            f"Workflow resume dispatched to Celery: {run_id}",
+            run_id=run_id,
+            task_id=task_result.id,
+        )
+
+        # Return None since the actual result will be available asynchronously
+        return None
+
+    async def schedule_resume(
+        self,
+        run_id: str,
+        storage: "StorageBackend",
+    ) -> None:
+        """
+        Schedule immediate workflow resumption via Celery task.
+
+        This is called by resume_hook() to trigger workflow resumption
+        after a hook event is received.
+        """
+        from pyworkflow.celery.tasks import resume_workflow_task
+
+        logger.info(
+            f"Scheduling workflow resume via Celery: {run_id}",
+            run_id=run_id,
+        )
+
+        storage_config = self._get_storage_config(storage)
+
+        resume_workflow_task.apply_async(
+            args=[run_id],
+            kwargs={"storage_config": storage_config},
+        )
+
+        logger.info(
+            f"Workflow resume scheduled: {run_id}",
+            run_id=run_id,
+        )
+
+    async def schedule_wake(
+        self,
+        run_id: str,
+        wake_time: datetime,
+        storage: "StorageBackend",
+    ) -> None:
+        """
+        Schedule workflow resumption at a specific time using Celery.
+
+        Uses Celery's countdown feature to delay task execution.
+        """
+        from pyworkflow.celery.tasks import schedule_workflow_resumption
+
+        logger.info(
+            f"Scheduling workflow wake via Celery: {run_id}",
+            run_id=run_id,
+            wake_time=wake_time.isoformat(),
+        )
+
+        # Use the existing schedule function which handles the delay calculation
+        schedule_workflow_resumption(run_id, wake_time)
+
+        logger.info(
+            f"Workflow wake scheduled: {run_id}",
+            run_id=run_id,
+            wake_time=wake_time.isoformat(),
+        )
+
+    async def start_child_workflow(
+        self,
+        workflow_func: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        child_run_id: str,
+        workflow_name: str,
+        storage: "StorageBackend",
+        parent_run_id: str,
+        child_id: str,
+        wait_for_completion: bool,
+    ) -> None:
+        """
+        Start a child workflow via Celery (fire-and-forget).
+
+        Dispatches child workflow execution to a Celery worker. The worker
+        will handle parent notification and resumption when the child completes.
+        """
+        from pyworkflow.celery.tasks import start_child_workflow_task
+        from pyworkflow.serialization.encoder import serialize_args, serialize_kwargs
+
+        logger.info(
+            f"Dispatching child workflow to Celery: {workflow_name}",
+            child_run_id=child_run_id,
+            parent_run_id=parent_run_id,
+            child_id=child_id,
+        )
+
+        # Serialize arguments for Celery transport
+        args_json = serialize_args(*args)
+        kwargs_json = serialize_kwargs(**kwargs)
+
+        # Get storage configuration for workers
+        storage_config = self._get_storage_config(storage)
+
+        # Dispatch to Celery worker
+        task_result = start_child_workflow_task.delay(
+            workflow_name=workflow_name,
+            args_json=args_json,
+            kwargs_json=kwargs_json,
+            child_run_id=child_run_id,
+            storage_config=storage_config,
+            parent_run_id=parent_run_id,
+            child_id=child_id,
+            wait_for_completion=wait_for_completion,
+        )
+
+        logger.info(
+            f"Child workflow dispatched to Celery: {workflow_name}",
+            child_run_id=child_run_id,
+            task_id=task_result.id,
+        )

pyworkflow/runtime/factory.py

@@ -0,0 +1,101 @@
+"""
+Runtime factory and registration.
+
+This module provides:
+- Runtime registration and lookup
+- Validation of runtime + durable combinations
+"""
+
+from pyworkflow.runtime.base import Runtime
+
+# Runtime registry
+_runtimes: dict[str, type[Runtime]] = {}
+
+
+def register_runtime(name: str, runtime_class: type[Runtime]) -> None:
+    """
+    Register a runtime implementation.
+
+    Args:
+        name: Runtime identifier (e.g., "local", "celery")
+        runtime_class: Runtime class to register
+
+    Example:
+        >>> from pyworkflow.runtime import register_runtime
+        >>> from myapp.runtime import CustomRuntime
+        >>> register_runtime("custom", CustomRuntime)
+    """
+    _runtimes[name] = runtime_class
+
+
+def get_runtime(name: str) -> Runtime:
+    """
+    Get a runtime instance by name.
+
+    Args:
+        name: Runtime identifier
+
+    Returns:
+        Runtime instance
+
+    Raises:
+        ValueError: If runtime is not registered
+    """
+    if name not in _runtimes:
+        available = ", ".join(sorted(_runtimes.keys())) or "(none registered)"
+        raise ValueError(f"Unknown runtime: '{name}'. Available runtimes: {available}")
+    return _runtimes[name]()
+
+
+def validate_runtime_durable(runtime: Runtime, durable: bool) -> None:
+    """
+    Validate that a runtime supports the requested durability mode.
+
+    Args:
+        runtime: Runtime instance
+        durable: Whether durable mode is requested
+
+    Raises:
+        ValueError: If the combination is not supported
+    """
+    if durable and not runtime.supports_durable:
+        raise ValueError(
+            f"Runtime '{runtime.name}' does not support durable workflows. "
+            f"Use durable=False or choose a different runtime."
+        )
+    if not durable and not runtime.supports_transient:
+        raise ValueError(
+            f"Runtime '{runtime.name}' requires durable=True. "
+            f"This runtime does not support transient workflows."
+        )
+
+
+def list_runtimes() -> dict[str, type[Runtime]]:
+    """
+    List all registered runtimes.
+
+    Returns:
+        Dictionary of runtime name -> runtime class
+    """
+    return dict(_runtimes)
+
+
+# Register built-in runtimes
+def _register_builtin_runtimes() -> None:
+    """Register built-in runtimes."""
+    from pyworkflow.runtime.local import LocalRuntime
+
+    register_runtime("local", LocalRuntime)
+
+    # Register Celery runtime (lazy import to avoid circular deps)
+    try:
+        from pyworkflow.runtime.celery import CeleryRuntime
+
+        register_runtime("celery", CeleryRuntime)
+    except ImportError:
+        # Celery not installed, skip registration
+        pass
+
+
+# Auto-register on import
+_register_builtin_runtimes()