pyworkflow-engine 0.2.12__tar.gz → 0.2.14__tar.gz

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyworkflow-engine
-Version: 0.2.12
+Version: 0.2.14
 Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
 Author: PyWorkflow Contributors
 License: MIT

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyproject.toml

@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
 
 [project]
 name = "pyworkflow-engine"
-version = "0.2.12"
+version = "0.2.14"
 description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
 readme = "README.md"
 requires-python = ">=3.11"

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyworkflow/__init__.py

@@ -29,7 +29,7 @@ Quick Start:
     >>> run_id = await start(my_workflow, "Alice")
 """
 
-__version__ = "0.2.12"
+__version__ = "0.2.14"
 
 # Configuration
 from pyworkflow.config import (

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyworkflow/celery/tasks.py

@@ -508,8 +508,17 @@ async def _record_step_completion_and_resume(
         run_id, EventType.STEP_COMPLETED.value, step_id=step_id
     )
     if already_completed:
+        # A prior worker recorded STEP_COMPLETED but may have crashed before
+        # scheduling the workflow resume. Re-schedule defensively — the resume
+        # task is idempotent via try_claim_run(SUSPENDED -> RUNNING).
+        schedule_workflow_resumption(
+            run_id,
+            datetime.now(UTC),
+            storage_config,
+            triggered_by="step_completed_idempotent_retry",
+        )
         logger.info(
-            "Step already completed by another task, skipping",
+            "Step already completed by another task, re-scheduled resume",
             run_id=run_id,
             step_id=step_id,
             step_name=step_name,
@@ -537,8 +546,16 @@ async def _record_step_completion_and_resume(
             run_id, EventType.STEP_COMPLETED.value, step_id=step_id
         )
         if already_completed:
+            # Defensive re-schedule: the peer that recorded STEP_COMPLETED may
+            # have crashed before scheduling resume. Safe due to try_claim_run.
+            schedule_workflow_resumption(
+                run_id,
+                datetime.now(UTC),
+                storage_config,
+                triggered_by="step_completed_idempotent_retry",
+            )
             logger.info(
-                "Step already completed by another task during wait, skipping",
+                "Step already completed by another task during wait, re-scheduled resume",
                 run_id=run_id,
                 step_id=step_id,
                 step_name=step_name,
@@ -676,8 +693,17 @@ async def _record_step_failure_and_resume(
         run_id, EventType.STEP_FAILED.value, step_id=step_id, is_retryable="False"
     )
     if already_completed or already_failed_terminal:
+        # A prior worker recorded the terminal event but may have crashed before
+        # scheduling the workflow resume. Re-schedule defensively — safe due to
+        # try_claim_run(SUSPENDED -> RUNNING) idempotency.
+        schedule_workflow_resumption(
+            run_id,
+            datetime.now(UTC),
+            storage_config,
+            triggered_by="step_failed_idempotent_retry",
+        )
         logger.info(
-            "Step already completed/failed by another task, skipping",
+            "Step already completed/failed by another task, re-scheduled resume",
             run_id=run_id,
             step_id=step_id,
             step_name=step_name,
@@ -708,8 +734,16 @@ async def _record_step_failure_and_resume(
             run_id, EventType.STEP_FAILED.value, step_id=step_id, is_retryable="False"
         )
         if already_completed or already_failed_terminal:
+            # Defensive re-schedule: peer that recorded the terminal event may
+            # have crashed before scheduling resume. Safe due to try_claim_run.
+            schedule_workflow_resumption(
+                run_id,
+                datetime.now(UTC),
+                storage_config,
+                triggered_by="step_failed_idempotent_retry",
+            )
             logger.info(
-                "Step already completed/failed by another task during wait, skipping",
+                "Step already completed/failed by another task during wait, re-scheduled resume",
                 run_id=run_id,
                 step_id=step_id,
                 step_name=step_name,

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyworkflow/streams/__init__.py

@@ -28,8 +28,10 @@ from pyworkflow.streams.checkpoint import (  # noqa: F401
 )
 from pyworkflow.streams.consumer import StreamConsumer, poll_once  # noqa: F401
 from pyworkflow.streams.context import (  # noqa: F401
+    acknowledge_inbox_signal,
     get_checkpoint,
     get_current_signal,
+    poll_inbox,
     save_checkpoint,
     set_result,
     suspend,
@@ -73,6 +75,8 @@ __all__ = [
     "get_current_signal",
     "get_checkpoint",
     "save_checkpoint",
+    "poll_inbox",
+    "acknowledge_inbox_signal",
     # Checkpoint
     "CheckpointBackend",
     "DefaultCheckpointBackend",

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyworkflow/streams/context.py

@@ -182,3 +182,76 @@ def set_stream_run_id(stream_run_id: str | None) -> Any:
     Returns a token that can be used to reset the value.
     """
     return _current_stream_run_id.set(stream_run_id)
+
+
+async def poll_inbox(
+    signal_types: list[str] | None = None,
+) -> list[Signal]:
+    """Get unacknowledged signals for the current running stream step.
+
+    Allows a step that is in ``running`` status to discover signals that
+    arrived while it was executing.  Uses the same storage query as the
+    post-execution drain mechanism (``get_pending_signals``) but can be
+    called from *within* the lifecycle function, so a long-running step
+    does not have to return first.
+
+    Signals are filtered to the current ``stream_run_id`` to prevent
+    cross-run leakage.
+
+    Args:
+        signal_types: Optional whitelist.  When provided, only signals
+            whose ``signal_type`` is in this list are returned.
+
+    Returns:
+        A list of :class:`Signal` objects ordered by sequence number.
+    """
+    from loguru import logger
+
+    storage = _current_storage.get()
+    stream_id = _current_stream_id.get()
+    step_run_id = _current_step_run_id.get()
+    stream_run_id = _current_stream_run_id.get()
+
+    if not storage or not stream_id or not step_run_id:
+        return []
+
+    try:
+        pending = await storage.get_pending_signals(stream_id, step_run_id)
+    except Exception:
+        logger.debug("[poll_inbox] get_pending_signals failed", exc_info=True)
+        return []
+
+    results: list[Signal] = []
+    for row in pending:
+        # Filter to current stream run
+        if stream_run_id and row.get("stream_run_id") != stream_run_id:
+            continue
+        # Optional signal type filter
+        if signal_types and row.get("signal_type") not in signal_types:
+            continue
+        results.append(
+            Signal(
+                signal_id=row["signal_id"],
+                stream_id=row["stream_id"],
+                signal_type=row["signal_type"],
+                payload=row.get("payload") or {},
+                sequence=row.get("sequence", 0),
+                source_run_id=row.get("source_run_id"),
+                stream_run_id=row.get("stream_run_id"),
+                metadata=row.get("metadata") or {},
+            )
+        )
+    return results
+
+
+async def acknowledge_inbox_signal(signal_id: str) -> None:
+    """Acknowledge a signal retrieved via :func:`poll_inbox`.
+
+    Once acknowledged the signal will not appear in subsequent
+    ``poll_inbox`` calls or in the post-execution drain, preventing
+    double processing.
+    """
+    storage = _current_storage.get()
+    step_run_id = _current_step_run_id.get()
+    if storage and step_run_id:
+        await storage.acknowledge_signal(signal_id, step_run_id)

{pyworkflow_engine-0.2.12 → pyworkflow_engine-0.2.14}/pyworkflow/streams/dispatcher.py

@@ -66,9 +66,18 @@ async def dispatch_signal(signal: Signal, storage: Any) -> None:
             )
 
     if not waiting_steps:
-        logger.warning(
+        # Determine whether any subscriptions are in "running" status.
+        # If so the signal is safely persisted in the signals table and
+        # will be picked up by poll_inbox() (called from within the
+        # running step) or by _drain_pending_signals_for_step() when the
+        # step transitions back to "waiting".  Use DEBUG to avoid noisy
+        # warnings for this expected scenario.
+        has_running = any(s.get("status") == "running" for s in (all_subs or []))
+        log_fn = logger.debug if has_running else logger.warning
+        log_fn(
             f"No waiting steps for {signal.signal_type} on {signal.stream_id} "
-            f"(stream_run_id={signal.stream_run_id})",
+            f"(stream_run_id={signal.stream_run_id})"
+            + (" — signal queued for running step inbox" if has_running else ""),
             stream_id=signal.stream_id,
             signal_type=signal.signal_type,
             stream_run_id=signal.stream_run_id,