pyworkflow-engine 0.1.13__tar.gz → 0.1.15__tar.gz

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyworkflow-engine
-Version: 0.1.13
+Version: 0.1.15
 Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
 Author: PyWorkflow Contributors
 License: MIT

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyproject.toml

@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
 
 [project]
 name = "pyworkflow-engine"
-version = "0.1.13"
+version = "0.1.15"
 description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
 readme = "README.md"
 requires-python = ">=3.11"

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/__init__.py

@@ -29,7 +29,7 @@ Quick Start:
     >>> run_id = await start(my_workflow, "Alice")
 """
 
-__version__ = "0.1.13"
+__version__ = "0.1.15"
 
 # Configuration
 from pyworkflow.config import (

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/celery/singleton.py

@@ -96,7 +96,7 @@ class SingletonWorkflowTask(Task):
     Features:
     - Redis-based lock prevents duplicate execution
     - Support for unique_on with nested dict/list access (e.g., "data.run_id")
-    - Retry-safe: locks NOT released on failure (prevents duplicate during retries)
+    - Retry-safe: lock released in on_retry callback to allow retry to acquire it
     - Lock released on success or when max retries exceeded
     - Time-based lock expiry as safety net
 
@@ -124,7 +124,7 @@ class SingletonWorkflowTask(Task):
 
     # Lock behavior
     release_lock_on_success: bool = True
-    release_lock_on_failure: bool = False  # Keep lock during retries
+    release_lock_on_failure: bool = False  # Only release on max retries exceeded
 
     # Celery task settings
     max_retries: int | None = None
@@ -360,9 +360,11 @@ class SingletonWorkflowTask(Task):
         kwargs: dict[str, Any],
         einfo: Any,
     ) -> None:
-        """Lock is retained during retry."""
+        """Release lock during retry to allow retry task to acquire it."""
+        # Release lock so retry can acquire it via apply_async()
+        self.release_lock(task_args=args, task_kwargs=kwargs)
         logger.warning(
-            f"Task {self.name} retrying (lock retained)",
+            f"Task {self.name} retrying (lock released for retry)",
             task_id=task_id,
             retry_count=self.request.retries,
         )

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/celery/tasks.py

@@ -321,7 +321,7 @@ def execute_step_task(
             # Use exponential backoff for unexpected errors
             countdown = _calculate_exponential_backoff(self.request.retries)
             logger.warning(
-                f"Step failed (unexpected): {step_name}, retrying in {countdown:.1f}s...",
+                f"Step failed (unexpected): {step_name}, retrying in {countdown:.1f}s...: {str(e)}",
                 run_id=run_id,
                 step_id=step_id,
                 error=str(e),
@@ -1704,6 +1704,7 @@ async def _start_workflow_on_worker(
 def resume_workflow_task(
     run_id: str,
     storage_config: dict[str, Any] | None = None,
+    triggered_by_hook_id: str | None = None,
 ) -> Any | None:
     """
     Resume a suspended workflow.
@@ -1714,6 +1715,9 @@ def resume_workflow_task(
     Args:
         run_id: Workflow run ID to resume
         storage_config: Storage backend configuration
+        triggered_by_hook_id: Optional hook ID that triggered this resume.
+                              Used to prevent spurious resumes when a workflow
+                              has already moved past the triggering hook.
 
     Returns:
         Workflow result if completed, None if suspended again
@@ -1727,13 +1731,18 @@ def resume_workflow_task(
         f"RESUME_WORKFLOW_TASK ENTRY: {run_id}",
         run_id=run_id,
         celery_task_id=resume_workflow_task.request.id,
+        triggered_by_hook_id=triggered_by_hook_id,
     )
 
     # Get storage backend
     storage = _get_storage_backend(storage_config)
 
     # Resume workflow directly on worker
-    result = run_async(_resume_workflow_on_worker(run_id, storage, storage_config))
+    result = run_async(
+        _resume_workflow_on_worker(
+            run_id, storage, storage_config, triggered_by_hook_id=triggered_by_hook_id
+        )
+    )
 
     if result is not None:
         logger.info(f"Workflow completed on worker: {run_id}")
@@ -1940,15 +1949,81 @@ async def _complete_pending_sleeps(
     return updated_events
 
 
+def _is_hook_still_relevant(hook_id: str, events: list[Any]) -> bool:
+    """
+    Check if a hook is still relevant for resuming the workflow.
+
+    A hook is "still relevant" if there are no newer hooks created after
+    this hook was received. This prevents spurious resumes when:
+    1. resume_hook() is called multiple times for the same hook
+    2. The workflow moved past the first resume and created a new hook
+    3. The duplicate resume task runs but the workflow is now waiting on a different hook
+
+    Args:
+        hook_id: The hook ID that triggered the resume
+        events: List of workflow events
+
+    Returns:
+        True if the hook is still relevant, False if workflow has moved past it
+    """
+    from pyworkflow.engine.events import EventType
+
+    # Sort events by sequence to process in order
+    sorted_events = sorted(events, key=lambda e: e.sequence or 0)
+
+    # Find the sequence number of HOOK_RECEIVED for this hook
+    hook_received_sequence = None
+    for event in sorted_events:
+        if event.type == EventType.HOOK_RECEIVED and event.data.get("hook_id") == hook_id:
+            hook_received_sequence = event.sequence
+            break
+
+    if hook_received_sequence is None:
+        # Hook was never received - shouldn't happen, but allow resume
+        logger.warning(
+            f"Hook {hook_id} was not found in HOOK_RECEIVED events, allowing resume",
+            hook_id=hook_id,
+        )
+        return True
+
+    # Check if there's a HOOK_CREATED event after this hook was received
+    # (indicating the workflow has moved past this hook and created a new one)
+    for event in sorted_events:
+        if event.type == EventType.HOOK_CREATED:
+            event_sequence = event.sequence or 0
+            if event_sequence > hook_received_sequence:
+                # There's a newer hook - this resume is stale
+                newer_hook_id = event.data.get("hook_id")
+                logger.debug(
+                    f"Found newer hook {newer_hook_id} (seq {event_sequence}) "
+                    f"after triggered hook {hook_id} (received at seq {hook_received_sequence})",
+                    hook_id=hook_id,
+                    newer_hook_id=newer_hook_id,
+                )
+                return False
+
+    # No newer hooks created - this resume is still relevant
+    return True
+
+
 async def _resume_workflow_on_worker(
     run_id: str,
     storage: StorageBackend,
     storage_config: dict[str, Any] | None = None,
+    triggered_by_hook_id: str | None = None,
 ) -> Any | None:
     """
     Internal function to resume workflow on Celery worker.
 
     This mirrors the logic from testing.py but runs on workers.
+
+    Args:
+        run_id: Workflow run ID to resume
+        storage: Storage backend
+        storage_config: Storage configuration for task dispatch
+        triggered_by_hook_id: Optional hook ID that triggered this resume.
+                              If provided, we verify the hook is still relevant
+                              before resuming to prevent spurious resumes.
     """
     from pyworkflow.core.exceptions import WorkflowNotFoundError
 
@@ -1983,6 +2058,22 @@ async def _resume_workflow_on_worker(
         )
         return None
 
+    # If this resume was triggered by a specific hook, verify the hook is still relevant.
+    # A hook is "stale" if the workflow has already moved past it (created a newer hook).
+    # This prevents spurious resumes from duplicate resume_hook() calls.
+    if triggered_by_hook_id:
+        events = await storage.get_events(run_id)
+        hook_still_relevant = _is_hook_still_relevant(triggered_by_hook_id, events)
+        if not hook_still_relevant:
+            logger.info(
+                f"Hook {triggered_by_hook_id} is no longer relevant (workflow moved past it), "
+                "skipping spurious resume",
+                run_id=run_id,
+                workflow_name=run.workflow_name,
+                triggered_by_hook_id=triggered_by_hook_id,
+            )
+            return None
+
     # Check for cancellation flag
     cancellation_requested = await storage.check_cancellation_flag(run_id)
 

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/context/local.py

@@ -114,6 +114,31 @@ class LocalContext(WorkflowContext):
             self._replay_events(event_log)
             self._is_replaying = False
 
+    def _extract_counter_from_id(self, id_string: str) -> int:
+        """Extract counter value from hook_id or sleep_id.
+
+        Formats:
+        - hook_{name}_{counter}
+        - sleep_{counter}_{duration}s
+
+        Args:
+            id_string: The hook_id or sleep_id string
+
+        Returns:
+            The counter value, or 0 if parsing fails
+        """
+        try:
+            parts = id_string.split("_")
+            if id_string.startswith("hook_"):
+                # hook_{name}_{counter} - counter is last part
+                return int(parts[-1])
+            elif id_string.startswith("sleep_"):
+                # sleep_{counter}_{duration}s - counter is second part
+                return int(parts[1])
+        except (ValueError, IndexError):
+            pass
+        return 0
+
     def _replay_events(self, events: list[Any]) -> None:
         """Replay events to restore state."""
         from pyworkflow.engine.events import EventType
@@ -142,6 +167,12 @@ class LocalContext(WorkflowContext):
                 payload = deserialize(event.data.get("payload"))
                 self._hook_results[hook_id] = payload
 
+            elif event.type == EventType.HOOK_CREATED:
+                # Track pending hooks for re-suspension
+                hook_id = event.data.get("hook_id")
+                if hook_id:
+                    self._pending_hooks[hook_id] = event.data
+
             elif event.type == EventType.STEP_RETRYING:
                 step_id = event.data.get("step_id")
                 self._retry_states[step_id] = {
@@ -893,6 +924,21 @@ class LocalContext(WorkflowContext):
             logger.debug(f"[replay] Hook {hook_id} already received")
             return self._hook_results[hook_id]
 
+        # Check if already pending (created but not yet received - replay mode)
+        # This prevents duplicate hook creation when workflow resumes
+        if hook_id in self._pending_hooks:
+            logger.debug(f"[replay] Hook {hook_id} already pending, re-suspending")
+            pending_data = self._pending_hooks[hook_id]
+            actual_token = pending_data.get("token")
+            # Call on_created callback if provided
+            if on_created is not None:
+                await on_created(actual_token)
+            raise SuspensionSignal(
+                reason=f"hook:{hook_id}",
+                hook_id=hook_id,
+                token=actual_token,
+            )
+
         # Generate composite token: run_id:hook_id
         from pyworkflow.primitives.resume_hook import create_hook_token
 

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/primitives/resume_hook.py

@@ -185,6 +185,7 @@ async def resume_hook(
         hook_id=hook_id,
         status=HookStatus.RECEIVED,
         payload=serialized_payload,
+        run_id=run_id,
     )
 
     # Schedule workflow resumption via configured runtime
@@ -195,7 +196,7 @@ async def resume_hook(
     runtime = get_runtime(config.default_runtime)
 
     try:
-        await runtime.schedule_resume(run_id, storage)
+        await runtime.schedule_resume(run_id, storage, triggered_by_hook_id=hook_id)
     except Exception as e:
         logger.warning(
             f"Failed to schedule workflow resumption: {e}",

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/runtime/base.py

@@ -97,6 +97,7 @@ class Runtime(ABC):
         self,
         run_id: str,
         storage: "StorageBackend",
+        triggered_by_hook_id: str | None = None,
     ) -> None:
         """
         Schedule a workflow to be resumed immediately.
@@ -109,6 +110,9 @@ class Runtime(ABC):
         Args:
             run_id: The run_id of the workflow to resume
             storage: Storage backend
+            triggered_by_hook_id: Optional hook ID that triggered this resume.
+                                  Used by distributed runtimes to prevent
+                                  spurious resumes from duplicate calls.
         """
         # Default implementation: no-op
         # Subclasses override if they support async scheduling

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/runtime/celery.py

@@ -202,25 +202,36 @@ class CeleryRuntime(Runtime):
         self,
         run_id: str,
         storage: "StorageBackend",
+        triggered_by_hook_id: str | None = None,
     ) -> None:
         """
         Schedule immediate workflow resumption via Celery task.
 
         This is called by resume_hook() to trigger workflow resumption
         after a hook event is received.
+
+        Args:
+            run_id: The workflow run ID to resume
+            storage: Storage backend for configuration
+            triggered_by_hook_id: Optional hook ID that triggered this resume.
+                                  Used to prevent spurious resumes from duplicate calls.
         """
         from pyworkflow.celery.tasks import resume_workflow_task
 
         logger.info(
             f"Scheduling workflow resume via Celery: {run_id}",
             run_id=run_id,
+            triggered_by_hook_id=triggered_by_hook_id,
         )
 
         storage_config = self._get_storage_config(storage)
 
         resume_workflow_task.apply_async(
             args=[run_id],
-            kwargs={"storage_config": storage_config},
+            kwargs={
+                "storage_config": storage_config,
+                "triggered_by_hook_id": triggered_by_hook_id,
+            },
         )
 
         logger.info(

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/runtime/local.py

@@ -507,16 +507,24 @@ class LocalRuntime(Runtime):
         self,
         run_id: str,
         storage: "StorageBackend",
+        triggered_by_hook_id: str | None = None,
     ) -> None:
         """
         Schedule immediate workflow resumption.
 
         For local runtime, this directly calls resume_workflow since
         execution happens in-process.
+
+        Args:
+            run_id: The workflow run ID to resume
+            storage: Storage backend
+            triggered_by_hook_id: Optional hook ID that triggered this resume.
+                                  Not used in local runtime (no queueing).
         """
         logger.info(
             f"Scheduling immediate workflow resume: {run_id}",
             run_id=run_id,
+            triggered_by_hook_id=triggered_by_hook_id,
         )
 
         try:

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/storage/base.py

@@ -291,12 +291,13 @@ class StorageBackend(ABC):
         pass
 
     @abstractmethod
-    async def get_hook(self, hook_id: str) -> Hook | None:
+    async def get_hook(self, hook_id: str, run_id: str | None = None) -> Hook | None:
         """
         Retrieve a hook by ID.
 
         Args:
             hook_id: Hook identifier
+            run_id: Run ID (required for composite key lookup in SQL backends)
 
         Returns:
             Hook if found, None otherwise
@@ -322,6 +323,7 @@ class StorageBackend(ABC):
         hook_id: str,
         status: HookStatus,
         payload: str | None = None,
+        run_id: str | None = None,
     ) -> None:
         """
         Update hook status and optionally payload.
@@ -330,6 +332,7 @@ class StorageBackend(ABC):
             hook_id: Hook identifier
             status: New status
             payload: JSON serialized payload (if received)
+            run_id: Run ID (required for composite key lookup in SQL backends)
         """
         pass
 

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/storage/cassandra.py

@@ -1072,29 +1072,31 @@ class CassandraStorageBackend(StorageBackend):
 
         session.execute(batch)
 
-    async def get_hook(self, hook_id: str) -> Hook | None:
-        """Retrieve a hook by ID."""
+    async def get_hook(self, hook_id: str, run_id: str | None = None) -> Hook | None:
+        """Retrieve a hook by ID (run_id allows skipping lookup table)."""
         session = self._ensure_connected()
 
-        # First lookup run_id
-        lookup = session.execute(
-            SimpleStatement(
-                "SELECT run_id FROM hooks_by_id WHERE hook_id = %s",
-                consistency_level=self.read_consistency,
-            ),
-            (hook_id,),
-        ).one()
+        if not run_id:
+            # First lookup run_id from lookup table
+            lookup = session.execute(
+                SimpleStatement(
+                    "SELECT run_id FROM hooks_by_id WHERE hook_id = %s",
+                    consistency_level=self.read_consistency,
+                ),
+                (hook_id,),
+            ).one()
 
-        if not lookup:
-            return None
+            if not lookup:
+                return None
+            run_id = lookup.run_id
 
-        # Then get full hook
+        # Get full hook
         row = session.execute(
             SimpleStatement(
                 "SELECT * FROM hooks WHERE run_id = %s AND hook_id = %s",
                 consistency_level=self.read_consistency,
             ),
-            (lookup.run_id, hook_id),
+            (run_id, hook_id),
         ).one()
 
         if not row:
@@ -1137,21 +1139,24 @@ class CassandraStorageBackend(StorageBackend):
         hook_id: str,
         status: HookStatus,
         payload: str | None = None,
+        run_id: str | None = None,
     ) -> None:
         """Update hook status and optionally payload."""
         session = self._ensure_connected()
 
-        # First lookup run_id
-        lookup = session.execute(
-            SimpleStatement(
-                "SELECT run_id FROM hooks_by_id WHERE hook_id = %s",
-                consistency_level=self.read_consistency,
-            ),
-            (hook_id,),
-        ).one()
+        if not run_id:
+            # First lookup run_id from lookup table
+            lookup = session.execute(
+                SimpleStatement(
+                    "SELECT run_id FROM hooks_by_id WHERE hook_id = %s",
+                    consistency_level=self.read_consistency,
+                ),
+                (hook_id,),
+            ).one()
 
-        if not lookup:
-            return
+            if not lookup:
+                return
+            run_id = lookup.run_id
 
         received_at = datetime.now(UTC) if status == HookStatus.RECEIVED else None
 
@@ -1164,7 +1169,7 @@ class CassandraStorageBackend(StorageBackend):
                 """,
                 consistency_level=self.write_consistency,
             ),
-            (status.value, payload, received_at, lookup.run_id, hook_id),
+            (status.value, payload, received_at, run_id, hook_id),
         )
 
     async def list_hooks(

{pyworkflow_engine-0.1.13 → pyworkflow_engine-0.1.15}/pyworkflow/storage/dynamodb.py

@@ -722,9 +722,9 @@ class DynamoDBStorageBackend(StorageBackend):
     async def create_hook(self, hook: Hook) -> None:
         """Create a hook record."""
         async with self._get_client() as client:
-            # Main hook item
+            # Main hook item (composite key: run_id + hook_id)
             item = {
-                "PK": f"HOOK#{hook.hook_id}",
+                "PK": f"HOOK#{hook.run_id}#{hook.hook_id}",
                 "SK": "#METADATA",
                 "entity_type": "hook",
                 "hook_id": hook.hook_id,
@@ -741,12 +741,13 @@ class DynamoDBStorageBackend(StorageBackend):
                 "GSI1SK": f"{hook.status.value}#{hook.created_at.isoformat()}",
             }
 
-            # Token lookup item
+            # Token lookup item (stores run_id and hook_id for lookup)
             token_item = {
                 "PK": f"TOKEN#{hook.token}",
-                "SK": f"HOOK#{hook.hook_id}",
+                "SK": f"HOOK#{hook.run_id}#{hook.hook_id}",
                 "entity_type": "hook_token",
                 "hook_id": hook.hook_id,
+                "run_id": hook.run_id,
             }
 
             # Write both items
@@ -759,16 +760,26 @@ class DynamoDBStorageBackend(StorageBackend):
                 Item=self._dict_to_item(token_item),
             )
 
-    async def get_hook(self, hook_id: str) -> Hook | None:
-        """Retrieve a hook by ID."""
+    async def get_hook(self, hook_id: str, run_id: str | None = None) -> Hook | None:
+        """Retrieve a hook by ID (requires run_id for composite key)."""
         async with self._get_client() as client:
-            response = await client.get_item(
-                TableName=self.table_name,
-                Key={
-                    "PK": {"S": f"HOOK#{hook_id}"},
-                    "SK": {"S": "#METADATA"},
-                },
-            )
+            if run_id:
+                response = await client.get_item(
+                    TableName=self.table_name,
+                    Key={
+                        "PK": {"S": f"HOOK#{run_id}#{hook_id}"},
+                        "SK": {"S": "#METADATA"},
+                    },
+                )
+            else:
+                # Fallback: try old format without run_id
+                response = await client.get_item(
+                    TableName=self.table_name,
+                    Key={
+                        "PK": {"S": f"HOOK#{hook_id}"},
+                        "SK": {"S": "#METADATA"},
+                    },
+                )
 
             item = response.get("Item")
             if not item:
@@ -779,7 +790,7 @@ class DynamoDBStorageBackend(StorageBackend):
     async def get_hook_by_token(self, token: str) -> Hook | None:
         """Retrieve a hook by its token."""
         async with self._get_client() as client:
-            # First get the hook_id from the token lookup item
+            # First get the hook_id and run_id from the token lookup item
             response = await client.query(
                 TableName=self.table_name,
                 KeyConditionExpression="PK = :pk",
@@ -792,13 +803,16 @@ class DynamoDBStorageBackend(StorageBackend):
                 return None
 
             hook_id = self._deserialize_value(items[0]["hook_id"])
-            return await self.get_hook(hook_id)
+            run_id_attr = items[0].get("run_id")
+            run_id = self._deserialize_value(run_id_attr) if run_id_attr else None
+            return await self.get_hook(hook_id, run_id)
 
     async def update_hook_status(
         self,
         hook_id: str,
         status: HookStatus,
         payload: str | None = None,
+        run_id: str | None = None,
     ) -> None:
         """Update hook status and optionally payload."""
         async with self._get_client() as client:
@@ -814,10 +828,12 @@ class DynamoDBStorageBackend(StorageBackend):
                 update_expr += ", received_at = :received_at"
                 expr_values[":received_at"] = {"S": datetime.now(UTC).isoformat()}
 
+            pk = f"HOOK#{run_id}#{hook_id}" if run_id else f"HOOK#{hook_id}"
+
             await client.update_item(
                 TableName=self.table_name,
                 Key={
-                    "PK": {"S": f"HOOK#{hook_id}"},
+                    "PK": {"S": pk},
                     "SK": {"S": "#METADATA"},
                 },
                 UpdateExpression=update_expr,