pyworkflow-engine 0.1.7__py3-none-any.whl

pyworkflow/config.py

@@ -0,0 +1,329 @@
+"""
+PyWorkflow configuration system.
+
+Provides global configuration for runtime, storage, and default settings.
+
+Configuration is loaded in this priority order:
+1. Values set via pyworkflow.configure() (highest priority)
+2. Values from pyworkflow.config.yaml in current directory
+3. Default values
+
+Usage:
+    >>> import pyworkflow
+    >>> pyworkflow.configure(
+    ...     default_runtime="local",
+    ...     default_durable=False,
+    ...     storage=InMemoryStorageBackend(),
+    ... )
+"""
+
+import warnings
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from pyworkflow.storage.base import StorageBackend
+
+
+def _load_yaml_config() -> dict[str, Any]:
+    """
+    Load configuration from pyworkflow.config.yaml in current directory.
+
+    Returns:
+        Configuration dictionary, empty dict if file not found
+    """
+    config_path = Path.cwd() / "pyworkflow.config.yaml"
+    if not config_path.exists():
+        return {}
+
+    try:
+        import yaml
+
+        with open(config_path) as f:
+            config = yaml.safe_load(f) or {}
+            return config
+    except ImportError:
+        return {}
+    except Exception:
+        return {}
+
+
+def _create_storage_from_config(storage_config: dict[str, Any]) -> Optional["StorageBackend"]:
+    """Create a storage backend from config dictionary."""
+    if not storage_config:
+        return None
+
+    from pyworkflow.storage.config import config_to_storage
+
+    return config_to_storage(storage_config)
+
+
+@dataclass
+class PyWorkflowConfig:
+    """
+    Global configuration for PyWorkflow.
+
+    Attributes:
+        default_runtime: Default runtime to use ("local", "celery", etc.)
+        default_durable: Whether workflows are durable by default
+        default_retries: Default number of retries for steps
+        default_recover_on_worker_loss: Whether to auto-recover on worker failure
+        default_max_recovery_attempts: Default max recovery attempts on worker failure
+        storage: Storage backend instance for durable workflows
+        celery_broker: Celery broker URL (for celery runtime)
+        aws_region: AWS region (for lambda runtimes)
+        event_soft_limit: Log warning when event count reaches this (default: 10000)
+        event_hard_limit: Fail workflow when event count reaches this (default: 50000)
+        event_warning_interval: Log warning every N events after soft limit (default: 100)
+    """
+
+    # Defaults (can be overridden per-workflow)
+    default_runtime: str = "local"
+    default_durable: bool = False
+    default_retries: int = 3
+
+    # Fault tolerance defaults
+    default_recover_on_worker_loss: bool | None = (
+        None  # None = True for durable, False for transient
+    )
+    default_max_recovery_attempts: int = 3
+
+    # Infrastructure (app-level only)
+    storage: Optional["StorageBackend"] = None
+    celery_broker: str | None = None
+    aws_region: str | None = None
+
+    # Event limit settings (WARNING: Do not modify unless you understand the implications)
+    # These limits prevent runaway workflows from consuming excessive resources
+    event_soft_limit: int = 10_000  # Log warning at this count
+    event_hard_limit: int = 50_000  # Fail workflow at this count
+    event_warning_interval: int = 100  # Log warning every N events after soft limit
+
+
+def _config_from_yaml() -> PyWorkflowConfig:
+    """Create a PyWorkflowConfig from YAML file settings."""
+    yaml_config = _load_yaml_config()
+
+    if not yaml_config:
+        return PyWorkflowConfig()
+
+    # Map YAML keys to config attributes
+    runtime = yaml_config.get("runtime", "local")
+    durable = runtime == "celery"  # Celery runtime defaults to durable
+
+    # Create storage from config
+    storage = _create_storage_from_config(yaml_config.get("storage", {}))
+
+    # Get celery broker
+    celery_config = yaml_config.get("celery", {})
+    celery_broker = celery_config.get("broker")
+
+    return PyWorkflowConfig(
+        default_runtime=runtime,
+        default_durable=durable,
+        storage=storage,
+        celery_broker=celery_broker,
+    )
+
+
+# Global singleton
+_config: PyWorkflowConfig | None = None
+_config_loaded_from_yaml: bool = False
+
+
+def configure(
+    *,
+    module: str | None = None,
+    discover: bool = True,
+    **kwargs: Any,
+) -> None:
+    """
+    Configure PyWorkflow defaults.
+
+    Args:
+        module: Python module path to discover workflows from (e.g., "myapp.workflows").
+            If provided and discover=True, the module will be imported to register
+            workflows decorated with @workflow.
+        discover: If True (default) and module is provided, automatically discover
+            and register workflows from the specified module.
+        default_runtime: Default runtime ("local", "celery", "lambda", "durable-lambda")
+        default_durable: Whether workflows are durable by default
+        default_retries: Default number of retries for steps
+        default_recover_on_worker_loss: Whether to auto-recover on worker failure
+            (None = True for durable, False for transient)
+        default_max_recovery_attempts: Max recovery attempts on worker failure
+        storage: Storage backend instance
+        celery_broker: Celery broker URL
+        aws_region: AWS region
+
+    Event Limit Settings (Advanced - modify with caution):
+        event_soft_limit: Log warning when event count reaches this (default: 10000)
+        event_hard_limit: Fail workflow when event count reaches this (default: 50000)
+        event_warning_interval: Log warning every N events after soft limit (default: 100)
+
+    WARNING: Modifying event limits is not recommended. These defaults are carefully
+    chosen to prevent runaway workflows from consuming excessive resources.
+
+    Example:
+        >>> import pyworkflow
+        >>> from pyworkflow.storage import InMemoryStorageBackend
+        >>>
+        >>> pyworkflow.configure(
+        ...     default_runtime="local",
+        ...     default_durable=True,
+        ...     storage=InMemoryStorageBackend(),
+        ... )
+
+        >>> # Configure with workflow discovery
+        >>> pyworkflow.configure(module="myapp.workflows")
+    """
+    global _config
+    if _config is None:
+        _config = PyWorkflowConfig()
+
+    # Warn if user is modifying event limits
+    event_limit_keys = {"event_soft_limit", "event_hard_limit", "event_warning_interval"}
+    modified_limits = event_limit_keys & set(kwargs.keys())
+    if modified_limits:
+        warnings.warn(
+            f"Modifying event limits ({', '.join(sorted(modified_limits))}) is not recommended. "
+            "These defaults are carefully chosen to prevent runaway workflows.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    for key, value in kwargs.items():
+        if hasattr(_config, key):
+            setattr(_config, key, value)
+        else:
+            valid_keys = list(PyWorkflowConfig.__dataclass_fields__.keys())
+            raise ValueError(
+                f"Unknown config option: {key}. Valid options: {', '.join(valid_keys)}"
+            )
+
+    # Auto-discover workflows if module is specified
+    if discover and module:
+        from pyworkflow.discovery import discover_workflows
+
+        discover_workflows(module_path=module)
+
+
+def configure_from_yaml(path: str | Path, discover: bool = True) -> None:
+    """
+    Configure PyWorkflow from a specific YAML file.
+
+    Unlike the automatic YAML loading in get_config(), this function:
+    - Requires an explicit path
+    - Raises FileNotFoundError if the file doesn't exist
+    - Raises ValueError if YAML parsing fails
+    - Optionally discovers workflows from the 'module' field in the YAML
+
+    Args:
+        path: Path to the YAML configuration file
+        discover: If True (default), automatically discover and register
+            workflows from the 'module' or 'modules' field in the YAML file.
+            Set to False to skip discovery.
+
+    Raises:
+        FileNotFoundError: If the specified file doesn't exist
+        ValueError: If the YAML file is invalid or cannot be parsed
+        ImportError: If PyYAML is not installed
+        DiscoveryError: If workflow module discovery fails (when discover=True)
+
+    Example:
+        >>> import pyworkflow
+        >>> pyworkflow.configure_from_yaml("/etc/pyworkflow/config.yaml")
+
+        >>> # Skip workflow discovery
+        >>> pyworkflow.configure_from_yaml("/etc/pyworkflow/config.yaml", discover=False)
+    """
+    global _config, _config_loaded_from_yaml
+
+    config_path = Path(path)
+
+    if not config_path.exists():
+        raise FileNotFoundError(f"PyWorkflow configuration file not found: {config_path}")
+
+    try:
+        import yaml
+    except ImportError:
+        raise ImportError(
+            "PyYAML is required for YAML configuration. Install it with: pip install pyyaml"
+        )
+
+    try:
+        with open(config_path) as f:
+            yaml_config = yaml.safe_load(f) or {}
+    except yaml.YAMLError as e:
+        raise ValueError(f"Invalid YAML in {config_path}: {e}")
+
+    # Map YAML keys to config attributes (same logic as _config_from_yaml)
+    runtime = yaml_config.get("runtime", "local")
+    durable = runtime == "celery"  # Celery runtime defaults to durable
+
+    # Create storage from config
+    storage = _create_storage_from_config(yaml_config.get("storage", {}))
+
+    # Get celery broker
+    celery_config = yaml_config.get("celery", {})
+    celery_broker = celery_config.get("broker")
+
+    _config = PyWorkflowConfig(
+        default_runtime=runtime,
+        default_durable=durable,
+        storage=storage,
+        celery_broker=celery_broker,
+    )
+    _config_loaded_from_yaml = True
+
+    # Auto-discover workflows if enabled
+    if discover:
+        from pyworkflow.discovery import discover_workflows
+
+        discover_workflows(config=yaml_config, config_path=config_path)
+
+
+def get_config() -> PyWorkflowConfig:
+    """
+    Get the current configuration.
+
+    If not yet configured, loads from pyworkflow.config.yaml if present,
+    otherwise creates default configuration.
+
+    Returns:
+        Current PyWorkflowConfig instance
+    """
+    global _config, _config_loaded_from_yaml
+    if _config is None:
+        # Try to load from YAML config file first
+        _config = _config_from_yaml()
+        _config_loaded_from_yaml = True
+    return _config
+
+
+def reset_config() -> None:
+    """
+    Reset configuration to defaults.
+
+    Primarily used for testing.
+    """
+    global _config, _config_loaded_from_yaml
+    _config = None
+    _config_loaded_from_yaml = False
+
+
+def get_storage() -> Optional["StorageBackend"]:
+    """
+    Get the configured storage backend.
+
+    Returns:
+        StorageBackend instance if configured, None otherwise
+
+    Example:
+        >>> import pyworkflow
+        >>> from pyworkflow.storage import InMemoryStorageBackend
+        >>> pyworkflow.configure(storage=InMemoryStorageBackend())
+        >>> storage = pyworkflow.get_storage()
+    """
+    return get_config().storage

pyworkflow/context/__init__.py

@@ -0,0 +1,63 @@
+"""
+Workflow Context - The unified interface for workflow execution.
+
+The context provides implicit access to workflow operations within execution.
+Uses Python's contextvars for implicit context passing (similar to Scala's implicits).
+
+Available Contexts:
+- LocalContext: In-process execution with optional event sourcing
+- AWSContext: AWS Durable Lambda Functions with automatic checkpointing
+- MockContext: For testing workflows without side effects
+
+Usage with implicit context:
+    from pyworkflow.context import get_context
+
+    async def my_step(order_id: str):
+        ctx = get_context()  # Implicitly available
+        ctx.log(f"Processing {order_id}")
+        return {"order_id": order_id}
+
+    @workflow()
+    async def my_workflow(order_id: str):
+        # Context is set automatically by @workflow
+        ctx = get_context()
+        result = await ctx.run(my_step, order_id)
+        await ctx.sleep("5m")
+        return result
+
+Usage with explicit context (context manager):
+    from pyworkflow.context import LocalContext
+
+    async with LocalContext(run_id="run_123", workflow_name="my_workflow") as ctx:
+        result = await ctx.run(my_step, "order_123")
+"""
+
+from pyworkflow.context.base import (
+    WorkflowContext,
+    get_context,
+    has_context,
+    reset_context,
+    set_context,
+)
+from pyworkflow.context.local import LocalContext
+from pyworkflow.context.mock import MockContext
+
+__all__ = [
+    # Base context and helpers
+    "WorkflowContext",
+    "get_context",
+    "has_context",
+    "set_context",
+    "reset_context",
+    # Context implementations
+    "LocalContext",
+    "MockContext",
+]
+
+# AWS context is optional - only available if aws-durable-execution-sdk installed
+try:
+    from pyworkflow.context.aws import AWSContext
+
+    __all__.append("AWSContext")
+except ImportError:
+    pass

pyworkflow/context/aws.py

@@ -0,0 +1,230 @@
+"""
+AWSContext - AWS Durable Lambda Functions execution context.
+
+This context wraps the AWS Durable Execution SDK to provide PyWorkflow's
+context interface while leveraging AWS native checkpointing and durability.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Any
+
+from loguru import logger
+
+from pyworkflow.context.base import StepFunction, WorkflowContext
+from pyworkflow.utils.duration import parse_duration
+
+if TYPE_CHECKING:
+    from aws_durable_execution_sdk_python import DurableContext
+
+
+class AWSContext(WorkflowContext):
+    """
+    AWS Durable Lambda Functions execution context.
+
+    This context wraps the AWS Durable Execution SDK's DurableContext,
+    translating PyWorkflow operations to AWS SDK calls:
+
+    - ctx.run() -> context.step()
+    - ctx.sleep() -> context.wait()
+    - ctx.wait_for_event() -> context.wait_for_callback()
+    - ctx.parallel() -> context.parallel()
+
+    AWS handles all checkpointing, replay, and durability automatically.
+
+    Example:
+        # Created by @aws_workflow decorator, not directly
+        @aws_workflow()
+        async def my_workflow(ctx: AWSContext, order_id: str):
+            result = await ctx.run(validate_order, order_id)
+            await ctx.sleep("5m")  # No compute charges!
+            return result
+    """
+
+    def __init__(
+        self,
+        aws_context: DurableContext,
+        run_id: str = "aws_run",
+        workflow_name: str = "aws_workflow",
+    ) -> None:
+        """
+        Initialize AWS context.
+
+        Args:
+            aws_context: The AWS DurableContext from Lambda handler
+            run_id: Run ID (extracted from Lambda or generated)
+            workflow_name: Workflow name
+        """
+        super().__init__(run_id=run_id, workflow_name=workflow_name)
+        self._aws_ctx = aws_context
+        self._step_counter = 0
+
+    @property
+    def aws_context(self) -> DurableContext:
+        """Get the underlying AWS DurableContext."""
+        return self._aws_ctx
+
+    # =========================================================================
+    # Step execution
+    # =========================================================================
+
+    async def run(
+        self,
+        func: StepFunction,
+        *args: Any,
+        name: str | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Execute a step with AWS checkpointing.
+
+        Uses AWS context.step() which provides:
+        - Automatic checkpointing before/after execution
+        - Replay support (returns cached result if already completed)
+        - Retry handling
+
+        Args:
+            func: Step function to execute
+            *args: Arguments for the function
+            name: Optional step name (used for checkpointing)
+            **kwargs: Keyword arguments
+
+        Returns:
+            Step result
+        """
+        step_name = name or getattr(func, "__name__", None)
+        if not step_name:
+            self._step_counter += 1
+            step_name = f"step_{self._step_counter}"
+
+        logger.debug(f"[aws] Running step: {step_name}")
+
+        def execute_step(_: Any) -> Any:
+            """Inner function for AWS context.step()."""
+            if asyncio.iscoroutinefunction(func):
+                # Run async function in event loop
+                try:
+                    loop = asyncio.get_running_loop()
+                except RuntimeError:
+                    loop = None
+
+                if loop is not None:
+                    # Already in async context - use thread
+                    import concurrent.futures
+
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future = executor.submit(asyncio.run, func(*args, **kwargs))
+                        return future.result()
+                else:
+                    return asyncio.run(func(*args, **kwargs))
+            else:
+                return func(*args, **kwargs)
+
+        # Use AWS context.step() for checkpointing
+        result = self._aws_ctx.step(execute_step, name=step_name)
+
+        logger.debug(f"[aws] Step completed: {step_name}")
+        return result
+
+    # =========================================================================
+    # Sleep
+    # =========================================================================
+
+    async def sleep(self, duration: str | int | float) -> None:
+        """
+        Sleep using AWS native wait (no compute charges).
+
+        Uses AWS context.wait() which:
+        - Suspends Lambda execution
+        - No charges during wait time
+        - Automatically resumes when duration elapses
+
+        Args:
+            duration: Sleep duration
+        """
+        duration_seconds = parse_duration(duration) if isinstance(duration, str) else int(duration)
+
+        logger.debug(f"[aws] Sleeping: {duration_seconds}s")
+
+        # Try to use AWS Duration, fall back to raw seconds for mock
+        try:
+            from aws_durable_execution_sdk_python.config import Duration
+
+            duration_obj = Duration.from_seconds(duration_seconds)
+        except ImportError:
+            # Using mock context
+            duration_obj = duration_seconds
+
+        self._aws_ctx.wait(duration_obj)
+
+        logger.debug(f"[aws] Sleep completed: {duration_seconds}s")
+
+    # =========================================================================
+    # Parallel execution
+    # =========================================================================
+
+    async def parallel(self, *tasks: Any) -> list[Any]:
+        """
+        Execute tasks in parallel using AWS context.parallel().
+
+        Note: AWS parallel() has a different signature - it takes functions
+        that receive a child context. For simplicity, we fall back to
+        asyncio.gather for the MVP.
+
+        Args:
+            *tasks: Coroutines to execute in parallel
+
+        Returns:
+            List of results
+        """
+        # For MVP, use asyncio.gather
+        # TODO: Use AWS context.parallel() for better checkpointing
+        return list(await asyncio.gather(*tasks))
+
+    # =========================================================================
+    # External events (callbacks)
+    # =========================================================================
+
+    async def wait_for_event(
+        self,
+        event_name: str,
+        timeout: str | int | None = None,
+    ) -> Any:
+        """
+        Wait for an external event using AWS callbacks.
+
+        Uses AWS context.create_callback() or context.wait_for_callback().
+
+        Args:
+            event_name: Name for the callback
+            timeout: Optional timeout
+
+        Returns:
+            Callback payload when received
+        """
+        logger.debug(f"[aws] Waiting for event: {event_name}")
+
+        # Parse timeout
+        timeout_seconds = None
+        if timeout:
+            timeout_seconds = parse_duration(timeout) if isinstance(timeout, str) else int(timeout)
+
+        try:
+            from aws_durable_execution_sdk_python.config import CallbackConfig
+
+            config = None
+            if timeout_seconds:
+                config = CallbackConfig(timeout_seconds=timeout_seconds)
+
+            callback = self._aws_ctx.create_callback(name=event_name, config=config)
+
+            # Return the callback result when available
+            result = callback.result()
+
+            logger.debug(f"[aws] Event received: {event_name}")
+            return result
+
+        except ImportError:
+            # Mock context - return mock data
+            return {"event": event_name, "mock": True}