pyworkflow-engine 0.1.11__py3-none-any.whl → 0.1.13__py3-none-any.whl

pyworkflow/cli/commands/worker.py

@@ -43,8 +43,8 @@ def worker() -> None:
     "--concurrency",
     "-c",
     type=int,
-    default=None,
-    help="Number of worker processes (default: auto-detect)",
+    default=1,
+    help="Number of worker processes (default: 1)",
 )
 @click.option(
     "--loglevel",
@@ -67,8 +67,8 @@ def worker() -> None:
 @click.option(
     "--pool",
     type=click.Choice(["prefork", "solo", "eventlet", "gevent"], case_sensitive=False),
-    default=None,
-    help="Worker pool type. Use 'solo' for debugging with breakpoints",
+    default="prefork",
+    help="Worker pool type (default: prefork). Use 'solo' for debugging with breakpoints",
 )
 @click.pass_context
 def run_worker(
@@ -153,15 +153,16 @@ def run_worker(
         os.getenv("PYWORKFLOW_CELERY_RESULT_BACKEND", "redis://localhost:6379/1"),
     )
 
+    # Worker processes always need logging enabled
+    from loguru import logger as loguru_logger
+
+    loguru_logger.enable("pyworkflow")
+
     print_info("Starting Celery worker...")
     print_info(f"Broker: {broker_url}")
     print_info(f"Queues: {', '.join(queues)}")
-
-    if concurrency:
-        print_info(f"Concurrency: {concurrency}")
-
-    if pool:
-        print_info(f"Pool: {pool}")
+    print_info(f"Concurrency: {concurrency}")
+    print_info(f"Pool: {pool}")
 
     try:
         # Discover workflows using CLI discovery (reads from --module, env var, or YAML config)
@@ -212,11 +213,10 @@ def run_worker(
             "worker",
             f"--loglevel={loglevel.upper()}",
             f"--queues={','.join(queues)}",
+            f"--concurrency={concurrency}",  # Always set (default: 1)
+            f"--pool={pool}",  # Always set (default: prefork)
         ]
 
-        if concurrency:
-            worker_args.append(f"--concurrency={concurrency}")
-
         if hostname:
             worker_args.append(f"--hostname={hostname}")
 
@@ -224,9 +224,6 @@ def run_worker(
             worker_args.append("--beat")
             worker_args.append("--scheduler=pyworkflow.celery.scheduler:PyWorkflowScheduler")
 
-        if pool:
-            worker_args.append(f"--pool={pool}")
-
         print_success("Worker starting...")
         print_info("Press Ctrl+C to stop")
         print_info("")

pyworkflow/config.py

@@ -391,6 +391,11 @@ def reset_config() -> None:
     _config = None
     _config_loaded_from_yaml = False
 
+    # Also clear the storage cache to ensure test isolation
+    from pyworkflow.storage.config import clear_storage_cache
+
+    clear_storage_cache()
+
 
 def get_storage() -> Optional["StorageBackend"]:
     """

pyworkflow/context/base.py

@@ -307,6 +307,10 @@ class WorkflowContext(ABC):
         """Get failure info for a step."""
         return None  # Default: no failure info
 
+    def is_step_in_progress(self, step_id: str) -> bool:
+        """Check if a step is currently in progress (dispatched but not completed)."""
+        return False  # Default: no in-progress tracking
+
     def should_execute_step(self, step_id: str) -> bool:
         """Check if step should be executed (not already completed)."""
         return True  # Default: always execute

pyworkflow/context/local.py

@@ -104,6 +104,10 @@ class LocalContext(WorkflowContext):
         # Step failure tracking (for handling failures during replay)
         self._step_failures: dict[str, dict[str, Any]] = {}
 
+        # Steps in progress (dispatched to Celery but not yet completed)
+        # Used to prevent re-dispatch during resume
+        self._steps_in_progress: set[str] = set()
+
         # Replay state if resuming
         if event_log:
             self._is_replaying = True
@@ -116,10 +120,18 @@ class LocalContext(WorkflowContext):
         from pyworkflow.serialization.decoder import deserialize
 
         for event in events:
-            if event.type == EventType.STEP_COMPLETED:
+            if event.type == EventType.STEP_STARTED:
+                # Track step as in-progress (dispatched but not completed)
+                step_id = event.data.get("step_id")
+                if step_id:
+                    self._steps_in_progress.add(step_id)
+
+            elif event.type == EventType.STEP_COMPLETED:
                 step_id = event.data.get("step_id")
                 result = deserialize(event.data.get("result"))
                 self._step_results[step_id] = result
+                # Step completed - no longer in progress
+                self._steps_in_progress.discard(step_id)
 
             elif event.type == EventType.SLEEP_COMPLETED:
                 sleep_id = event.data.get("sleep_id")
@@ -156,6 +168,8 @@ class LocalContext(WorkflowContext):
                         "error_type": event.data.get("error_type", "Exception"),
                         "is_retryable": is_retryable,
                     }
+                    # Terminal failure - no longer in progress
+                    self._steps_in_progress.discard(step_id)
 
             elif event.type == EventType.CANCELLATION_REQUESTED:
                 self._cancellation_requested = True
@@ -335,6 +349,18 @@ class LocalContext(WorkflowContext):
             "is_retryable": is_retryable,
         }
 
+    def is_step_in_progress(self, step_id: str) -> bool:
+        """
+        Check if a step is currently in progress (dispatched but not completed).
+
+        This is used to prevent re-dispatching a step that's already running
+        on a Celery worker during workflow resume.
+
+        Returns:
+            True if step has STEP_STARTED but no STEP_COMPLETED/terminal STEP_FAILED
+        """
+        return step_id in self._steps_in_progress
+
     # =========================================================================
     # Sleep state management (for @step decorator and EventReplayer compatibility)
     # =========================================================================

pyworkflow/context/step_context.py

@@ -176,8 +176,7 @@ async def set_step_context(ctx: StepContext) -> None:
     Set the current step context and persist to storage.
 
     This function can only be called from workflow code, not from within steps.
-    When called, the context is persisted to storage and a CONTEXT_UPDATED event
-    is recorded for deterministic replay.
+    When called, the context is persisted to storage for resumption.
 
     Args:
         ctx: The StepContext instance to set
@@ -213,15 +212,6 @@ async def set_step_context(ctx: StepContext) -> None:
     if has_context():
         workflow_ctx = get_context()
         if workflow_ctx.is_durable and workflow_ctx.storage is not None:
-            from pyworkflow.engine.events import create_context_updated_event
-
-            # Record CONTEXT_UPDATED event for replay
-            event = create_context_updated_event(
-                run_id=workflow_ctx.run_id,
-                context_data=ctx.to_dict(),
-            )
-            await workflow_ctx.storage.record_event(event)
-
             # Update the WorkflowRun.context field
             await workflow_ctx.storage.update_run_context(workflow_ctx.run_id, ctx.to_dict())
 

pyworkflow/core/step.py

@@ -23,6 +23,7 @@ from loguru import logger
 from pyworkflow.context import get_context, has_context
 from pyworkflow.core.exceptions import FatalError, RetryableError
 from pyworkflow.core.registry import register_step
+from pyworkflow.core.validation import validate_step_parameters
 from pyworkflow.engine.events import (
     create_step_completed_event,
     create_step_failed_event,
@@ -118,6 +119,8 @@ def step(
                     f"Step {step_name} in transient mode, executing directly",
                     run_id=ctx.run_id,
                 )
+                # Validate parameters before execution
+                validate_step_parameters(func, args, kwargs, step_name)
                 return await _execute_with_retries(
                     func, args, kwargs, step_name, max_retries, retry_delay
                 )
@@ -126,22 +129,24 @@ def step(
             # Generate step ID (deterministic based on name + args)
             step_id = _generate_step_id(step_name, args, kwargs)
 
+            # Check if step has already failed (must check BEFORE cached result check)
+            # A failed step has no cached result, so should_execute_step would return True
+            # and skip this check if it were inside the should_execute_step block
+            if ctx.has_step_failed(step_id):
+                error_info = ctx.get_step_failure(step_id)
+                logger.error(
+                    f"Step {step_name} failed on remote worker",
+                    run_id=ctx.run_id,
+                    step_id=step_id,
+                    error=error_info.get("error") if error_info else "Unknown error",
+                )
+                raise FatalError(
+                    f"Step {step_name} failed: "
+                    f"{error_info.get('error') if error_info else 'Unknown error'}"
+                )
+
             # Check if step has already completed (replay)
             if not ctx.should_execute_step(step_id):
-                # Check if step failed (for distributed step dispatch)
-                if ctx.has_step_failed(step_id):
-                    error_info = ctx.get_step_failure(step_id)
-                    logger.error(
-                        f"Step {step_name} failed on remote worker",
-                        run_id=ctx.run_id,
-                        step_id=step_id,
-                        error=error_info.get("error") if error_info else "Unknown error",
-                    )
-                    raise FatalError(
-                        f"Step {step_name} failed: "
-                        f"{error_info.get('error') if error_info else 'Unknown error'}"
-                    )
-
                 logger.debug(
                     f"Step {step_name} already completed, using cached result",
                     run_id=ctx.run_id,
@@ -149,10 +154,29 @@ def step(
                 )
                 return ctx.get_step_result(step_id)
 
+            # Check if step is already in progress (dispatched to Celery but not completed)
+            # This prevents re-dispatch during resume when step is still running/retrying
+            if ctx.is_step_in_progress(step_id):
+                logger.debug(
+                    f"Step {step_name} already in progress, waiting for completion",
+                    run_id=ctx.run_id,
+                    step_id=step_id,
+                )
+                # Re-suspend and wait for existing task to complete
+                from pyworkflow.core.exceptions import SuspensionSignal
+
+                raise SuspensionSignal(
+                    reason=f"step_dispatch:{step_id}",
+                    step_id=step_id,
+                    step_name=step_name,
+                )
+
             # ========== Distributed Step Dispatch ==========
             # When running in a distributed runtime (e.g., Celery), dispatch steps
             # to step workers instead of executing inline.
             if ctx.runtime == "celery":
+                # Validate parameters before dispatching to Celery
+                validate_step_parameters(func, args, kwargs, step_name)
                 return await _dispatch_step_to_celery(
                     ctx=ctx,
                     func=func,
@@ -221,6 +245,9 @@ def step(
             # Check for cancellation before executing step
             ctx.check_cancellation()
 
+            # Validate parameters before execution
+            validate_step_parameters(func, args, kwargs, step_name)
+
             try:
                 # Execute step function
                 result = await func(*args, **kwargs)
@@ -366,12 +393,13 @@ def step(
                     )
 
                     # Record final STEP_FAILED event
+                    # is_retryable=False since we've exhausted all retries
                     failure_event = create_step_failed_event(
                         run_id=ctx.run_id,
                         step_id=step_id,
                         error=str(e),
                         error_type=type(e).__name__,
-                        is_retryable=True,
+                        is_retryable=False,
                         attempt=current_attempt,
                     )
                     await ctx.storage.record_event(failure_event)  # type: ignore[union-attr]

pyworkflow/core/validation.py

@@ -0,0 +1,112 @@
+"""
+Pydantic validation for step parameters.
+
+Validates step function arguments against their type hints using Pydantic's
+TypeAdapter for runtime type checking.
+"""
+
+import inspect
+from collections.abc import Callable
+from typing import Any, get_type_hints
+
+from pydantic import TypeAdapter, ValidationError
+
+from pyworkflow.core.exceptions import FatalError
+
+
+class StepValidationError(FatalError):
+    """
+    Raised when step parameter validation fails.
+
+    This is a FatalError subclass to ensure validation failures
+    immediately fail the workflow without retries.
+    """
+
+    def __init__(
+        self,
+        step_name: str,
+        param_name: str,
+        expected_type: type,
+        received_value: Any,
+        validation_error: ValidationError,
+    ) -> None:
+        self.step_name = step_name
+        self.param_name = param_name
+        self.expected_type = expected_type
+        self.received_value = received_value
+        self.validation_error = validation_error
+
+        # Build clear error message
+        error_details = str(validation_error)
+        message = (
+            f"Step '{step_name}' parameter validation failed for '{param_name}': "
+            f"expected {expected_type}, got {type(received_value).__name__} "
+            f"with value {received_value!r}. Details: {error_details}"
+        )
+        super().__init__(message)
+
+
+def validate_step_parameters(
+    func: Callable,
+    args: tuple,
+    kwargs: dict,
+    step_name: str,
+) -> None:
+    """
+    Validate step parameters against their type hints using Pydantic.
+
+    Only parameters with type annotations are validated. Parameters without
+    type hints are skipped.
+
+    Args:
+        func: The step function (original, unwrapped)
+        args: Positional arguments passed to the step
+        kwargs: Keyword arguments passed to the step
+        step_name: Name of the step for error messages
+
+    Raises:
+        StepValidationError: If any typed parameter fails validation
+    """
+    # Get function signature and type hints
+    sig = inspect.signature(func)
+
+    try:
+        # Try to get type hints, may fail for some edge cases
+        type_hints = get_type_hints(func)
+    except Exception:
+        # If we can't get type hints, skip validation
+        return
+
+    if not type_hints:
+        # No type hints at all, skip validation
+        return
+
+    # Bind arguments to parameters
+    try:
+        bound = sig.bind(*args, **kwargs)
+        bound.apply_defaults()
+    except TypeError:
+        # If binding fails, the function call itself will fail
+        # Let the normal execution handle this
+        return
+
+    # Validate each parameter that has a type hint
+    for param_name, param_value in bound.arguments.items():
+        if param_name not in type_hints:
+            # No type hint for this parameter, skip validation
+            continue
+
+        expected_type = type_hints[param_name]
+
+        try:
+            # Use Pydantic TypeAdapter for validation
+            adapter = TypeAdapter(expected_type)
+            adapter.validate_python(param_value)
+        except ValidationError as e:
+            raise StepValidationError(
+                step_name=step_name,
+                param_name=param_name,
+                expected_type=expected_type,
+                received_value=param_value,
+                validation_error=e,
+            )

pyworkflow/engine/events.py

@@ -24,6 +24,7 @@ class EventType(Enum):
     WORKFLOW_PAUSED = "workflow.paused"
     WORKFLOW_RESUMED = "workflow.resumed"
     WORKFLOW_CONTINUED_AS_NEW = "workflow.continued_as_new"  # Workflow continued with fresh history
+    WORKFLOW_SUSPENDED = "workflow.suspended"  # Workflow suspended (waiting for step/sleep/hook)
 
     # Step lifecycle events
     STEP_STARTED = "step.started"
@@ -45,9 +46,6 @@ class EventType(Enum):
     # Cancellation events
     CANCELLATION_REQUESTED = "cancellation.requested"
 
-    # Context events
-    CONTEXT_UPDATED = "context.updated"
-
     # Child workflow events
     CHILD_WORKFLOW_STARTED = "child_workflow.started"
     CHILD_WORKFLOW_COMPLETED = "child_workflow.completed"
@@ -174,6 +172,49 @@ def create_workflow_continued_as_new_event(
     )
 
 
+def create_workflow_suspended_event(
+    run_id: str,
+    reason: str,
+    step_id: str | None = None,
+    step_name: str | None = None,
+    sleep_id: str | None = None,
+    hook_id: str | None = None,
+    child_id: str | None = None,
+) -> Event:
+    """
+    Create a workflow suspended event.
+
+    This event is recorded when a workflow suspends execution, typically
+    waiting for a step to complete on a worker, a sleep to elapse, a hook
+    to be received, or a child workflow to complete.
+
+    Args:
+        run_id: The workflow run ID
+        reason: Suspension reason (e.g., "step_dispatch:step_id", "sleep", "hook", "child_workflow")
+        step_id: Step ID if suspended for step execution
+        step_name: Step name if suspended for step execution
+        sleep_id: Sleep ID if suspended for sleep
+        hook_id: Hook ID if suspended for webhook
+        child_id: Child workflow ID if suspended for child
+
+    Returns:
+        Event: The workflow suspended event
+    """
+    return Event(
+        run_id=run_id,
+        type=EventType.WORKFLOW_SUSPENDED,
+        data={
+            "reason": reason,
+            "step_id": step_id,
+            "step_name": step_name,
+            "sleep_id": sleep_id,
+            "hook_id": hook_id,
+            "child_id": child_id,
+            "suspended_at": datetime.now(UTC).isoformat(),
+        },
+    )
+
+
 def create_workflow_interrupted_event(
     run_id: str,
     reason: str,
@@ -483,33 +524,6 @@ def create_step_cancelled_event(
     )
 
 
-def create_context_updated_event(
-    run_id: str,
-    context_data: dict[str, Any],
-) -> Event:
-    """
-    Create a context updated event.
-
-    This event is recorded when set_step_context() is called in workflow code.
-    It captures the full context state for deterministic replay.
-
-    Args:
-        run_id: The workflow run ID
-        context_data: The serialized context data (from StepContext.to_dict())
-
-    Returns:
-        Event: The context updated event
-    """
-    return Event(
-        run_id=run_id,
-        type=EventType.CONTEXT_UPDATED,
-        data={
-            "context": context_data,
-            "updated_at": datetime.now(UTC).isoformat(),
-        },
-    )
-
-
 # Child workflow event creation helpers
 
 

pyworkflow/engine/executor.py

@@ -283,9 +283,29 @@ async def _execute_workflow_local(
         return result
 
     except SuspensionSignal as e:
-        # Workflow suspended (sleep or hook)
+        # Workflow suspended (sleep, hook, or step dispatch)
         await storage.update_run_status(run_id=run_id, status=RunStatus.SUSPENDED)
 
+        # Record WORKFLOW_SUSPENDED event
+        from pyworkflow.engine.events import create_workflow_suspended_event
+
+        step_id = e.data.get("step_id") if e.data else None
+        step_name = e.data.get("step_name") if e.data else None
+        sleep_id = e.data.get("sleep_id") if e.data else None
+        hook_id = e.data.get("hook_id") if e.data else None
+        child_id = e.data.get("child_id") if e.data else None
+
+        suspended_event = create_workflow_suspended_event(
+            run_id=run_id,
+            reason=e.reason,
+            step_id=step_id,
+            step_name=step_name,
+            sleep_id=sleep_id,
+            hook_id=hook_id,
+            child_id=child_id,
+        )
+        await storage.record_event(suspended_event)
+
         logger.info(
             f"Workflow suspended: {e.reason}",
             run_id=run_id,

pyworkflow/engine/replay.py

@@ -93,9 +93,6 @@ class EventReplayer:
         elif event.type == EventType.CANCELLATION_REQUESTED:
             await self._apply_cancellation_requested(ctx, event)
 
-        elif event.type == EventType.CONTEXT_UPDATED:
-            await self._apply_context_updated(ctx, event)
-
         # Other event types don't affect replay state
         # (workflow_started, step_started, step_failed, etc. are informational)
 
@@ -258,42 +255,6 @@ class EventReplayer:
             requested_by=requested_by,
         )
 
-    async def _apply_context_updated(self, ctx: LocalContext, event: Event) -> None:
-        """
-        Apply context_updated event - restore step context.
-
-        During replay, this restores the step context to its state at the time
-        the event was recorded. This ensures deterministic replay.
-        """
-        from pyworkflow.context.step_context import (
-            _set_step_context_internal,
-            get_step_context_class,
-        )
-
-        context_data = event.data.get("context", {})
-
-        if context_data:
-            # Get the registered context class
-            context_class = get_step_context_class()
-            if context_class is not None:
-                try:
-                    step_ctx = context_class.from_dict(context_data)
-                    _set_step_context_internal(step_ctx)
-                    logger.debug(
-                        "Restored step context from replay",
-                        run_id=ctx.run_id,
-                    )
-                except Exception as e:
-                    logger.warning(
-                        f"Failed to restore step context: {e}",
-                        run_id=ctx.run_id,
-                    )
-            else:
-                logger.debug(
-                    "No context class registered, skipping context restoration",
-                    run_id=ctx.run_id,
-                )
-
 
 # Singleton instance
 _replayer = EventReplayer()

pyworkflow/observability/logging.py

@@ -5,13 +5,40 @@ Provides structured logging with context-aware formatting for workflows, steps,
 and events. Integrates with loguru for powerful logging capabilities.
 """
 
+import logging
 import sys
 from pathlib import Path
+from types import FrameType
 from typing import Any
 
 from loguru import logger
 
 
+class InterceptHandler(logging.Handler):
+    """
+    Intercept standard logging calls and redirect them to loguru.
+
+    This ensures all logs (including from third-party libraries like Celery)
+    go through loguru's unified formatting.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        # Get corresponding Loguru level if it exists
+        try:
+            level: str | int = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller from where originated the logged message
+        frame_or_none: FrameType | None = logging.currentframe()
+        depth = 2
+        while frame_or_none is not None and frame_or_none.f_code.co_filename == logging.__file__:
+            frame_or_none = frame_or_none.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
 def configure_logging(
     level: str = "INFO",
     log_file: str | None = None,
@@ -50,6 +77,18 @@ def configure_logging(
     # Remove default logger
     logger.remove()
 
+    # Intercept standard library logging and redirect to loguru
+    # This ensures Celery and other libraries' logs go through loguru
+    logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)
+
+    # Suppress verbose Celery logs - only show warnings and above
+    # PyWorkflow provides its own task execution logs
+    for logger_name in ("celery", "celery.task", "celery.worker", "kombu", "amqp"):
+        celery_logger = logging.getLogger(logger_name)
+        celery_logger.handlers = [InterceptHandler()]
+        celery_logger.propagate = False
+        celery_logger.setLevel(logging.WARNING)
+
     # Console format
     if json_logs:
         # JSON format for structured logging
@@ -90,13 +129,16 @@ def configure_logging(
         record["extra"]["_context"] = extra_str
         return True
 
+    # Add console handler - use stdout for better Celery worker compatibility
+    # enqueue=True makes it process-safe for multiprocessing (Celery workers)
     logger.add(
-        sys.stderr,
+        sys.stdout,
         format=console_format + "{extra[_context]}",
         level=level,
         colorize=not json_logs,
         serialize=json_logs,
         filter=format_with_context,  # type: ignore[arg-type]
+        enqueue=True,  # Process-safe logging for Celery workers
     )
 
     # Add file handler if requested

pyworkflow/runtime/celery.py

@@ -248,7 +248,7 @@ class CeleryRuntime(Runtime):
         )
 
         # Use the existing schedule function which handles the delay calculation
-        schedule_workflow_resumption(run_id, wake_time)
+        schedule_workflow_resumption(run_id, wake_time, triggered_by="celery_runtime_schedule_wake")
 
         logger.info(
             f"Workflow wake scheduled: {run_id}",