pyworkflow-engine 0.2.2__tar.gz → 0.2.5__tar.gz

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyworkflow-engine
-Version: 0.2.2
+Version: 0.2.5
 Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
 Author: PyWorkflow Contributors
 License: MIT

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyproject.toml

@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
 
 [project]
 name = "pyworkflow-engine"
-version = "0.2.2"
+version = "0.2.5"
 description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
 readme = "README.md"
 requires-python = ">=3.11"

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/__init__.py

@@ -29,7 +29,7 @@ Quick Start:
     >>> run_id = await start(my_workflow, "Alice")
 """
 
-__version__ = "0.2.2"
+__version__ = "0.2.5"
 
 # Configuration
 from pyworkflow.config import (

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/celery/app.py

@@ -304,33 +304,17 @@ def create_celery_app(
         # Broker transport options - prevent task redelivery
         # See: https://github.com/celery/celery/issues/5935
         "broker_transport_options": final_broker_opts,
-        # Task routing
+        # Task routing — use direct exchange (each task specifies queue= explicitly)
         "task_default_queue": "pyworkflow.default",
         "task_default_exchange": "pyworkflow",
-        "task_default_exchange_type": "topic",
-        "task_default_routing_key": "workflow.default",
+        "task_default_exchange_type": "direct",
+        "task_default_routing_key": "pyworkflow.default",
         # Task queues
         "task_queues": (
-            Queue(
-                "pyworkflow.default",
-                Exchange("pyworkflow", type="topic"),
-                routing_key="workflow.#",
-            ),
-            Queue(
-                "pyworkflow.steps",
-                Exchange("pyworkflow", type="topic"),
-                routing_key="workflow.step.#",
-            ),
-            Queue(
-                "pyworkflow.workflows",
-                Exchange("pyworkflow", type="topic"),
-                routing_key="workflow.workflow.#",
-            ),
-            Queue(
-                "pyworkflow.schedules",
-                Exchange("pyworkflow", type="topic"),
-                routing_key="workflow.schedule.#",
-            ),
+            Queue("pyworkflow.default", Exchange("pyworkflow", type="direct")),
+            Queue("pyworkflow.steps", Exchange("pyworkflow", type="direct")),
+            Queue("pyworkflow.workflows", Exchange("pyworkflow", type="direct")),
+            Queue("pyworkflow.schedules", Exchange("pyworkflow", type="direct")),
         ),
         # Task execution
         "task_acks_late": True,

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/celery/tasks.py

@@ -10,7 +10,6 @@ These tasks enable:
 """
 
 import asyncio
-import contextlib
 import random
 import uuid
 from collections.abc import Callable
@@ -186,29 +185,6 @@ def execute_step_task(
         )
         return None
 
-    # Execution-time guard: prevent duplicate step execution on concurrent workers.
-    # The singleton lock (unique_on=["run_id", "step_id"]) prevents duplicate queueing,
-    # but broker-level re-delivery or race conditions can still result in two workers
-    # executing the same step. This Redis lock ensures only one proceeds.
-    _exec_lock_key = f"pyworkflow:step_exec:{run_id}:{step_id}"
-    _exec_lock_backend = self.singleton_backend
-    _exec_lock_acquired = False
-    if _exec_lock_backend is not None:
-        _exec_lock_acquired = _exec_lock_backend.lock(
-            _exec_lock_key, self.request.id or "unknown", expiry=self._lock_expiry
-        )
-        if not _exec_lock_acquired:
-            logger.warning(
-                "Step already being executed by another worker, skipping duplicate",
-                run_id=run_id,
-                step_id=step_id,
-                step_name=step_name,
-                existing_worker_task=_exec_lock_backend.get(_exec_lock_key),
-            )
-            return None
-    else:
-        _exec_lock_acquired = True  # No Redis = no locking
-
     # Deserialize arguments
     args = deserialize_args(args_json)
     kwargs = deserialize_kwargs(kwargs_json)
@@ -475,11 +451,6 @@ def execute_step_task(
             raise FatalError(f"Step '{step_name}' failed after retries: {str(e)}") from e
 
     finally:
-        # Release execution lock so retries or other tasks can proceed.
-        if _exec_lock_acquired and _exec_lock_backend is not None:
-            with contextlib.suppress(Exception):
-                _exec_lock_backend.unlock(_exec_lock_key)
-
         # Clean up step execution context (checkpoint/hook primitives)
         if step_exec_tokens is not None:
             from pyworkflow.primitives.step_checkpoint import reset_step_execution_context
@@ -864,62 +835,33 @@ def start_workflow_task(
         celery_task_id=self.request.id,
     )
 
-    # Execution-time guard: prevent duplicate workflow start on concurrent workers.
-    # The singleton lock (unique_on=["run_id"]) prevents duplicate queueing, but
-    # broker-level re-delivery or race conditions can still result in two workers
-    # executing the same start task. This Redis lock ensures only one proceeds.
-    _exec_lock_key = f"pyworkflow:wf_start:{run_id}"
-    _exec_lock_backend = self.singleton_backend
-    _exec_lock_acquired = False
-    if _exec_lock_backend is not None:
-        _exec_lock_acquired = _exec_lock_backend.lock(
-            _exec_lock_key, self.request.id or "unknown", expiry=self._lock_expiry
-        )
-        if not _exec_lock_acquired:
-            logger.warning(
-                "Workflow start already being executed by another worker, skipping duplicate",
-                run_id=run_id,
-                workflow_name=workflow_name,
-                existing_worker_task=_exec_lock_backend.get(_exec_lock_key),
-            )
-            return run_id
-    else:
-        _exec_lock_acquired = True  # No Redis = no locking
-
-    try:
-        # Get workflow metadata
-        workflow_meta = get_workflow(workflow_name)
-        if not workflow_meta:
-            raise ValueError(f"Workflow '{workflow_name}' not found in registry")
+    # Get workflow metadata
+    workflow_meta = get_workflow(workflow_name)
+    if not workflow_meta:
+        raise ValueError(f"Workflow '{workflow_name}' not found in registry")
 
-        # Deserialize arguments
-        args = deserialize_args(args_json)
-        kwargs = deserialize_kwargs(kwargs_json)
+    # Deserialize arguments
+    args = deserialize_args(args_json)
+    kwargs = deserialize_kwargs(kwargs_json)
 
-        # Get storage backend
-        storage = _get_storage_backend(storage_config)
+    # Get storage backend
+    storage = _get_storage_backend(storage_config)
 
-        # Execute workflow directly on worker
-        result_run_id = run_async(
-            _start_workflow_on_worker(
-                workflow_meta=workflow_meta,
-                args=args,
-                kwargs=kwargs,
-                storage=storage,
-                storage_config=storage_config,
-                idempotency_key=idempotency_key,
-                run_id=run_id,
-            )
+    # Execute workflow directly on worker
+    result_run_id = run_async(
+        _start_workflow_on_worker(
+            workflow_meta=workflow_meta,
+            args=args,
+            kwargs=kwargs,
+            storage=storage,
+            storage_config=storage_config,
+            idempotency_key=idempotency_key,
+            run_id=run_id,
         )
+    )
 
-        logger.info(f"Workflow execution initiated: {workflow_name}", run_id=result_run_id)
-        return result_run_id
-
-    finally:
-        # Release execution lock so retries or re-dispatches can proceed.
-        if _exec_lock_acquired and _exec_lock_backend is not None:
-            with contextlib.suppress(Exception):
-                _exec_lock_backend.unlock(_exec_lock_key)
+    logger.info(f"Workflow execution initiated: {workflow_name}", run_id=result_run_id)
+    return result_run_id
 
 
 @celery_app.task(
@@ -2520,6 +2462,9 @@ async def _resume_workflow_on_worker(
                     run_id=run_id,
                     step_id=step_id,
                 )
+                resume_workflow_task.release_lock(
+                    task_args=[run_id], task_kwargs={"storage_config": storage_config}
+                )
                 schedule_workflow_resumption(
                     run_id,
                     datetime.now(UTC),
@@ -2539,6 +2484,9 @@ async def _resume_workflow_on_worker(
                     run_id=run_id,
                     hook_id=hook_id,
                 )
+                resume_workflow_task.release_lock(
+                    task_args=[run_id], task_kwargs={"storage_config": storage_config}
+                )
                 schedule_workflow_resumption(
                     run_id,
                     datetime.now(UTC),
@@ -2550,6 +2498,13 @@ async def _resume_workflow_on_worker(
         # Schedule automatic resumption if we have a resume_at time
         resume_at = e.data.get("resume_at") if e.data else None
         if resume_at:
+            # Release the singleton lock BEFORE scheduling the next resume.
+            # Without this, the new resume_workflow_task.apply_async() sees
+            # the lock still held by the current (calling) task and silently
+            # drops the schedule as a duplicate.
+            resume_workflow_task.release_lock(
+                task_args=[run_id], task_kwargs={"storage_config": storage_config}
+            )
             schedule_workflow_resumption(
                 run_id, resume_at, storage_config=storage_config, triggered_by="start_sleep_hook"
             )

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/context/local.py

@@ -151,7 +151,16 @@ class LocalContext(WorkflowContext):
         from pyworkflow.engine.events import EventType
         from pyworkflow.serialization.decoder import deserialize
 
+        max_counter = 0
         for event in events:
+            # Track step counter from sleep and hook IDs to avoid collisions
+            for key in ("sleep_id", "hook_id"):
+                id_val = event.data.get(key) if event.data else None
+                if id_val:
+                    c = self._extract_counter_from_id(id_val)
+                    if c > max_counter:
+                        max_counter = c
+
             if event.type == EventType.STEP_STARTED:
                 # Track step as in-progress (dispatched but not completed)
                 step_id = event.data.get("step_id")
@@ -271,6 +280,10 @@ class LocalContext(WorkflowContext):
                     }
                     self._pending_children.pop(child_id, None)
 
+        # Restore step counter so new sleeps/hooks don't collide with replayed ones
+        if max_counter > self._step_counter:
+            self._step_counter = max_counter
+
     @property
     def is_durable(self) -> bool:
         return self._durable

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/core/step.py

@@ -669,7 +669,8 @@ async def _dispatch_step_to_celery(
 
     # Defense-in-depth: check if STEP_STARTED was already recorded for this step.
     # This guards against duplicate dispatch when two resume tasks race and both
-    # replay past the same step. If already started, re-suspend to wait.
+    # replay past the same step. If already started, re-suspend to wait for
+    # the running task to complete via task_reject_on_worker_lost re-delivery.
     # Exception: if STEP_SUSPENDED was recorded (step_hook suspension), the step
     # needs re-dispatch so it can re-execute and find the HOOK_RECEIVED event.
     events = await ctx.storage.get_events(ctx.run_id)
@@ -682,7 +683,7 @@ async def _dispatch_step_to_celery(
     )
     if already_started and not was_suspended:
         logger.info(
-            f"Step {step_name} already has STEP_STARTED event, re-suspending",
+            f"Step {step_name} already has STEP_STARTED event, re-suspending (task running)",
             run_id=ctx.run_id,
             step_id=step_id,
         )

{pyworkflow_engine-0.2.2 → pyworkflow_engine-0.2.5}/pyworkflow/storage/base.py

@@ -755,6 +755,7 @@ class StorageBackend(ABC):
         signal_type: str,
         payload: dict,
         source_run_id: str | None = None,
+        stream_run_id: str | None = None,
         metadata: dict | None = None,
     ) -> int:
         """
@@ -766,6 +767,7 @@ class StorageBackend(ABC):
             signal_type: Signal type (e.g., "task.created")
             payload: Signal payload data
             source_run_id: Optional source workflow run ID
+            stream_run_id: Optional stream run identifier for grouping signals
             metadata: Optional signal metadata
 
         Returns:
@@ -798,6 +800,40 @@ class StorageBackend(ABC):
         """
         raise NotImplementedError("This storage backend does not support streams")
 
+    async def query_stream_signals(
+        self,
+        stream_id: str,
+        stream_run_id: str,
+        *,
+        source_run_id: str | None = None,
+        signal_type: str | None = None,
+        after_sequence: int | None = None,
+        before_sequence: int | None = None,
+        last_n: int | None = None,
+        limit: int = 50,
+    ) -> list[dict]:
+        """
+        Query signals from a stream with rich filtering.
+
+        Args:
+            stream_id: Stream identifier.
+            source_run_id: Filter to signals from a specific workflow run.
+            stream_run_id: Filter to signals from a specific stream run.
+            signal_type: Filter by signal type (e.g. ``"task.completed"``).
+            after_sequence: Only include signals with ``sequence >= value``.
+            before_sequence: Only include signals with ``sequence <= value``.
+            last_n: Return the N most recent signals (overrides
+                ``after_sequence`` and ``limit``).
+            limit: Maximum number of signals to return.
+
+        Returns:
+            List of signal dicts ordered by sequence ASC.
+
+        Raises:
+            NotImplementedError: If backend doesn't support streams.
+        """
+        raise NotImplementedError("This storage backend does not support streams")
+
     # Subscription Operations
 
     async def register_stream_subscription(
@@ -839,6 +875,28 @@ class StorageBackend(ABC):
         """
         raise NotImplementedError("This storage backend does not support streams")
 
+    async def get_subscriptions_for_stream(
+        self,
+        stream_id: str,
+        signal_type: str,
+    ) -> list[dict]:
+        """
+        Get ALL subscriptions for a signal type on a stream, regardless of status.
+
+        Unlike ``get_waiting_steps`` which only returns ``"waiting"``
+        subscriptions, this returns subscriptions in any status (waiting,
+        running, etc.).  Used to avoid creating duplicate subscriptions
+        when a step is currently executing.
+
+        Args:
+            stream_id: Stream identifier
+            signal_type: Signal type to match
+
+        Returns:
+            List of dicts with step subscription info
+        """
+        raise NotImplementedError("This storage backend does not support streams")
+
     async def update_subscription_status(
         self,
         stream_id: str,