durable-workflow 2.0.3__tar.gz → 2.1.0__tar.gz

{durable_workflow-2.0.3/src/durable_workflow.egg-info → durable_workflow-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.0.3
+Version: 2.1.0
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.0.3"
+version = "2.1.0"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -71,9 +71,9 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.0.3"
-registry-version = "2.0.3"
-supported-server-versions = "2.0.0"
+product-train = "2.1.0"
+registry-version = "2.1.0"
+supported-server-versions = "2.4.0"
 worker-protocol-version = "1.19"
 control-plane-version = "2"
 durable-selection = true

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow/client.py

@@ -20,8 +20,10 @@ from __future__ import annotations
 
 import asyncio
 import hashlib
+import json as json_module
 import math
 import os
+import re
 import time
 import uuid
 import warnings
@@ -30,7 +32,7 @@ from dataclasses import dataclass
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as _pkg_version
 from typing import Any
-from urllib.parse import quote, urlencode, urlsplit
+from urllib.parse import quote, unquote, urlencode, urlsplit
 
 import httpx
 
@@ -101,6 +103,26 @@ _RUNTIME_EXTERNAL_PAYLOAD_FETCH_PATH_TEMPLATE = (
     "/external-payloads/v1/{referenceId}"
 )
 _RUNTIME_EXTERNAL_PAYLOAD_ERROR_BODY_LIMIT = 64 * 1024
+_PAYLOAD_COMPLETION_SCHEMA = "durable-workflow.v2.payload-completion-context.v1"
+_PAYLOAD_COMPLETION_HEADER = "X-Durable-Workflow-Payload-Completion"
+
+
+def _payload_completion_context(path: str, body: Any) -> dict[str, Any] | None:
+    match = re.fullmatch(r"/worker/(activity|workflow|query)-tasks/([^/]+)/(complete|fail)", path.split("?")[0])
+    if match is None or not isinstance(body, dict):
+        return None
+    kind, task_id, operation = match.groups()
+    attempt = body.get("activity_attempt_id" if kind == "activity" else f"{kind}_task_attempt")
+    owner = body.get("lease_owner")
+    if not isinstance(owner, str) or not owner:
+        return None
+    if kind == "activity":
+        if not isinstance(attempt, str) or not attempt:
+            return None
+    elif type(attempt) is not int or attempt < 1:
+        return None
+    return {"schema": _PAYLOAD_COMPLETION_SCHEMA, "kind": kind, "task_id": unquote(task_id),
+            "attempt": attempt, "lease_owner": owner, "operation": operation}
 
 
 def _default_sdk_version() -> str:
@@ -277,6 +299,7 @@ class _RuntimeExternalPayloadTransport:
     max_payload_bytes: int
     request_timeout_seconds: float
     status: str
+    completion_context: bool = False
 
 
 @dataclass
@@ -577,6 +600,7 @@ class WorkflowCommandResult:
     command_status: str | None = None
     command_id: str | None = None
     raw: dict[str, Any] | None = None
+    run_id: str | None = None
 
     @classmethod
     def from_dict(cls, data: dict[str, Any], *, workflow_id: str | None = None) -> WorkflowCommandResult:
@@ -586,6 +610,7 @@ class WorkflowCommandResult:
             command_status=data.get("command_status"),
             command_id=data.get("command_id"),
             raw=data,
+            run_id=data.get("run_id"),
         )
 
 
@@ -1302,6 +1327,12 @@ class WorkflowHandle:
         """Ask the server to repair this workflow. See :meth:`Client.repair_workflow`."""
         return await self._client.repair_workflow(self.workflow_id)
 
+    async def redrive(self, *, request_id: str | None = None) -> WorkflowCommandResult:
+        """Continue this failed run from its recorded activity failure boundary."""
+        if self.run_id is None:
+            raise ValueError("run_id is required to redrive a workflow from a handle")
+        return await self._client.redrive_workflow(self.workflow_id, self.run_id, request_id=request_id)
+
     async def archive(self, *, reason: str | None = None) -> WorkflowCommandResult:
         """Move this terminal workflow into the archive tier. See :meth:`Client.archive_workflow`."""
         return await self._client.archive_workflow(self.workflow_id, reason=reason)
@@ -1734,6 +1765,9 @@ class Client:
                     worker=worker,
                     transport=transport,
                     uploaded={},
+                    completion=(
+                        _payload_completion_context(path, json) if worker and transport.completion_context else None
+                    ),
                 )
 
         start = time.perf_counter()
@@ -1904,11 +1938,15 @@ class Client:
             )
 
         status = policy.get("status")
+        completion = upload.get("completion_context")
         transport = _RuntimeExternalPayloadTransport(
             threshold_bytes=threshold_bytes,
             max_payload_bytes=max_payload_bytes,
             request_timeout_seconds=float(request_timeout_seconds),
             status=status if isinstance(status, str) else "unknown",
+            completion_context=isinstance(completion, dict)
+            and completion.get("schema") == _PAYLOAD_COMPLETION_SCHEMA
+            and completion.get("header") == _PAYLOAD_COMPLETION_HEADER,
         )
         self._runtime_external_payload_transport_cache = transport
         self._runtime_external_payload_transport_resolved = True
@@ -1972,6 +2010,8 @@ class Client:
         worker: bool,
         transport: _RuntimeExternalPayloadTransport,
         uploaded: dict[tuple[str, str, int], RuntimeExternalPayloadReference],
+        completion: dict[str, Any] | None = None,
+        slot: tuple[str | int, ...] = (),
     ) -> Any:
         if isinstance(value, dict):
             if (
@@ -1984,6 +2024,8 @@ class Client:
                     worker=worker,
                     transport=transport,
                     uploaded=uploaded,
+                    completion=completion,
+                    slot=(*slot, "result"),
                 )
                 if "external_payload" in externalized_result:
                     normalized_command["result"] = externalized_result
@@ -2014,6 +2056,7 @@ class Client:
                         sha256=sha256,
                         worker=worker,
                         transport=transport,
+                        completion={**completion, "slot": list(slot)} if completion is not None else None,
                     )
                     uploaded[identity] = reference
                 return {"codec": codec, "external_payload": reference.to_dict()}
@@ -2024,6 +2067,8 @@ class Client:
                     worker=worker,
                     transport=transport,
                     uploaded=uploaded,
+                    completion=completion,
+                    slot=(*slot, key),
                 )
                 for key, item in value.items()
             }
@@ -2045,8 +2090,10 @@ class Client:
                     worker=worker,
                     transport=transport,
                     uploaded=uploaded,
+                    completion=completion,
+                    slot=(*slot, index),
                 )
-                for item in value
+                for index, item in enumerate(value)
             ]
         return value
 
@@ -2058,6 +2105,7 @@ class Client:
         sha256: str,
         worker: bool,
         transport: _RuntimeExternalPayloadTransport,
+        completion: dict[str, Any] | None = None,
     ) -> RuntimeExternalPayloadReference:
         headers = self._headers(worker=worker)
         headers.update({
@@ -2069,13 +2117,30 @@ class Client:
         })
 
         async def _do_request() -> httpx.Response:
+            attempt_headers = dict(headers)
             response = await self._http.request(
                 "POST",
                 f"/api{_RUNTIME_EXTERNAL_PAYLOAD_UPLOAD_PATH}",
-                headers=headers,
+                headers=attempt_headers,
                 content=data,
                 timeout=transport.request_timeout_seconds,
             )
+            if (response.status_code == 503 and completion is not None
+                    and len(response.content) <= _RUNTIME_EXTERNAL_PAYLOAD_ERROR_BODY_LIMIT):
+                try:
+                    refusal = response.json()
+                except ValueError:
+                    refusal = None
+                context = json_module.dumps(completion, separators=(",", ":"))
+                if (isinstance(refusal, dict) and refusal.get("reason") == "storage_pressure"
+                        and refusal.get("storage_state") == "draining" and len(context.encode("utf-8")) <= 4096):
+                    # Keep the same identity if the ordinary transport policy retries
+                    # an ambiguous response; never rerun application activity code.
+                    attempt_headers[_PAYLOAD_COMPLETION_HEADER] = context
+                    response = await self._http.request(
+                        "POST", f"/api{_RUNTIME_EXTERNAL_PAYLOAD_UPLOAD_PATH}", headers=attempt_headers,
+                        content=data, timeout=transport.request_timeout_seconds,
+                    )
             response.raise_for_status()
             return response
 
@@ -4076,6 +4141,23 @@ class Client:
         data = await self._request("POST", f"/workflows/{workflow_id}/repair", json={}, context=workflow_id)
         return WorkflowCommandResult.from_dict(data, workflow_id=workflow_id)
 
+    async def redrive_workflow(
+        self,
+        workflow_id: str,
+        failed_run_id: str,
+        *,
+        request_id: str | None = None,
+    ) -> WorkflowCommandResult:
+        """Continue a failed run, reusing its completed activity results."""
+        body = {"request_id": request_id} if request_id is not None else {}
+        data = await self._request(
+            "POST",
+            f"/workflows/{workflow_id}/runs/{failed_run_id}/redrive",
+            json=body,
+            context=workflow_id,
+        )
+        return WorkflowCommandResult.from_dict(data, workflow_id=workflow_id)
+
     async def archive_workflow(self, workflow_id: str, *, reason: str | None = None) -> WorkflowCommandResult:
         """Move a terminal workflow into the archive tier, returning the command outcome."""
         body: dict[str, Any] = {}

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow/errors.py

@@ -551,6 +551,7 @@ class ActivityFailed(DurableWorkflowError):
         message: str,
         *,
         activity_type: str | None = None,
+        step_sequence: int | None = None,
         activity_execution_id: str | None = None,
         activity_attempt_id: str | None = None,
         failure_id: str | None = None,
@@ -564,6 +565,7 @@ class ActivityFailed(DurableWorkflowError):
     ) -> None:
         super().__init__(message)
         self.activity_type = activity_type
+        self.step_sequence = step_sequence
         self.activity_execution_id = activity_execution_id
         self.activity_attempt_id = activity_attempt_id
         self.failure_id = failure_id

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow/retry_policy.py

@@ -43,6 +43,12 @@ def _storage_refusal(exc: Exception) -> tuple[ServerError, str | None] | None:
         body = exc.response.json()
     except ValueError:
         return None
+    # A payload upload is content-addressed and precedes completion submission.
+    # Even a late pressure refusal can retry those same bytes. This local retry
+    # classification does not alter the original response exposed to callers.
+    if (isinstance(body, dict) and "request_admitted" not in body and exc.request.method == "POST"
+            and exc.request.url.path.endswith("/api/external-payloads/v1")):
+        body = {**body, "request_admitted": False}
     error = ServerError(exc.response.status_code, body)
     if error.reason() not in ("storage_pressure", "storage_admission_unavailable"):
         return None

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow/sync.py

@@ -134,6 +134,10 @@ class SyncWorkflowHandle:
         result: WorkflowCommandResult = self._runner.run(self._handle.repair())
         return result
 
+    def redrive(self, *, request_id: str | None = None) -> WorkflowCommandResult:
+        result: WorkflowCommandResult = self._runner.run(self._handle.redrive(request_id=request_id))
+        return result
+
     def archive(self, *, reason: str | None = None) -> WorkflowCommandResult:
         result: WorkflowCommandResult = self._runner.run(self._handle.archive(reason=reason))
         return result
@@ -525,6 +529,18 @@ class Client:
         result: WorkflowCommandResult = self._runner.run(self._async.repair_workflow(workflow_id))
         return result
 
+    def redrive_workflow(
+        self,
+        workflow_id: str,
+        failed_run_id: str,
+        *,
+        request_id: str | None = None,
+    ) -> WorkflowCommandResult:
+        result: WorkflowCommandResult = self._runner.run(
+            self._async.redrive_workflow(workflow_id, failed_run_id, request_id=request_id)
+        )
+        return result
+
     def archive_workflow(self, workflow_id: str, *, reason: str | None = None) -> WorkflowCommandResult:
         result: WorkflowCommandResult = self._runner.run(self._async.archive_workflow(workflow_id, reason=reason))
         return result

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow/workflow.py

@@ -621,6 +621,8 @@ class FailWorkflow:
     exception_class: str | None = None
     exception: dict[str, Any] | None = None
     non_retryable: bool = False
+    failed_step_sequence: int | None = None
+    failed_activity_execution_id: str | None = None
 
     def to_server_command(
         self,
@@ -642,6 +644,9 @@ class FailWorkflow:
             cmd["exception"] = self.exception
         if self.non_retryable:
             cmd["non_retryable"] = True
+        if self.failed_step_sequence is not None and self.failed_activity_execution_id is not None:
+            cmd["failed_step_sequence"] = self.failed_step_sequence
+            cmd["failed_activity_execution_id"] = self.failed_activity_execution_id
         return cmd
 
 
@@ -2795,11 +2800,25 @@ def _fail_workflow_from_exception(exc: BaseException, *, prefix: str | None = No
         if activity_failure.activity_attempt_id is not None:
             exception["activity_attempt_id"] = activity_failure.activity_attempt_id
 
+    failed_step_sequence = None
+    failed_activity_execution_id = None
+    if (
+        isinstance(exc, ActivityFailed)
+        and type(exc.step_sequence) is int
+        and exc.step_sequence > 0
+        and isinstance(exc.activity_execution_id, str)
+        and exc.activity_execution_id
+    ):
+        failed_step_sequence = exc.step_sequence
+        failed_activity_execution_id = exc.activity_execution_id
+
     return FailWorkflow(
         message=message,
         exception_type=exception_type,
         exception_class=exception_class,
         exception=exception,
+        failed_step_sequence=failed_step_sequence,
+        failed_activity_execution_id=failed_activity_execution_id,
     )
 
 
@@ -3345,6 +3364,11 @@ def _activity_failed_from_payload(payload: Mapping[str, Any]) -> ActivityFailed:
     return ActivityFailed(
         message or "activity failed",
         activity_type=_activity_type_from_payload(payload),
+        step_sequence=(
+            payload["sequence"]
+            if type(payload.get("sequence")) is int and payload["sequence"] > 0
+            else None
+        ),
         activity_execution_id=_optional_str(payload.get("activity_execution_id")),
         activity_attempt_id=_optional_str(payload.get("activity_attempt_id")),
         failure_id=_optional_str(payload.get("failure_id")),

{durable_workflow-2.0.3 → durable_workflow-2.1.0/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.0.3
+Version: 2.1.0
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/src/durable_workflow.egg-info/SOURCES.txt

@@ -61,6 +61,7 @@ tests/test_pypi_project_surface.py
 tests/test_python_conformance.py
 tests/test_queries.py
 tests/test_readme_quickstart.py
+tests/test_redrive_replay.py
 tests/test_release_metadata.py
 tests/test_replay.py
 tests/test_replay_conformance.py

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_client.py

@@ -832,14 +832,20 @@ class TestWorkflowHandleControlPlane:
     async def test_maintenance_delegates_to_client(self, client: Client) -> None:
         handle = WorkflowHandle(client, workflow_id="wf-1", run_id="r1", workflow_type="greeter")
         client.repair_workflow = AsyncMock(return_value="repair")
+        client.redrive_workflow = AsyncMock(return_value="redrive")
         client.archive_workflow = AsyncMock(return_value="archive")
 
         assert await handle.repair() == "repair"
+        assert await handle.redrive(request_id="retry-1") == "redrive"
         assert await handle.archive(reason="retention") == "archive"
 
         client.repair_workflow.assert_awaited_once_with("wf-1")
+        client.redrive_workflow.assert_awaited_once_with("wf-1", "r1", request_id="retry-1")
         client.archive_workflow.assert_awaited_once_with("wf-1", reason="retention")
 
+        with pytest.raises(ValueError, match="run_id is required"):
+            await WorkflowHandle(client, workflow_id="wf-1").redrive()
+
 
 class TestSignalWorkflow:
     @pytest.mark.asyncio
@@ -1065,6 +1071,38 @@ class TestTerminateWorkflow:
 
 
 class TestWorkflowMaintenanceCommands:
+    @pytest.mark.asyncio
+    async def test_redrive_returns_successor_and_sends_optional_request_id(self, client: Client) -> None:
+        resp = _mock_response(202, {
+            "workflow_id": "wf-1",
+            "run_id": "run-2",
+            "outcome": "redriven",
+            "command_status": "accepted",
+            "resume_step_sequence": 2,
+        })
+        with patch.object(client._http, "request", new_callable=AsyncMock, return_value=resp) as mock:
+            result = await client.redrive_workflow("wf-1", "run-1", request_id="retry-1")
+
+        assert mock.call_args.args == ("POST", "/api/workflows/wf-1/runs/run-1/redrive")
+        assert mock.call_args.kwargs["json"] == {"request_id": "retry-1"}
+        assert result.workflow_id == "wf-1"
+        assert result.run_id == "run-2"
+        assert result.outcome == "redriven"
+        assert result.raw is not None and result.raw["resume_step_sequence"] == 2
+
+        with patch.object(client._http, "request", new_callable=AsyncMock, return_value=resp) as mock:
+            await client.redrive_workflow("wf-1", "run-1")
+        assert mock.call_args.kwargs["json"] == {}
+
+    @pytest.mark.asyncio
+    async def test_redrive_rejection_is_not_reported_as_success(self, client: Client) -> None:
+        resp = _mock_response(409, {"reason": "run_not_failed", "message": "Run is not failed."})
+        with (
+            patch.object(client._http, "request", new_callable=AsyncMock, return_value=resp),
+            pytest.raises(ServerError),
+        ):
+            await client.redrive_workflow("wf-1", "run-1")
+
     @pytest.mark.asyncio
     async def test_repair_request_matches_polyglot_fixture(self, client: Client) -> None:
         fixture_path = Path(__file__).parent / "fixtures" / "control-plane" / "workflow-repair-parity.json"

durable_workflow-2.1.0/tests/test_redrive_replay.py

@@ -0,0 +1,46 @@
+from durable_workflow import serializer, workflow
+from durable_workflow.workflow import CompleteWorkflow, ScheduleActivity, replay
+
+
+@workflow.defn(name="tests.redrive")
+class RedriveWorkflow:
+    def run(self, ctx):  # type: ignore[no-untyped-def]
+        first = yield ctx.schedule_activity("tests.first", [])
+        second = yield ctx.schedule_activity("tests.second", [first])
+        return {"first": first, "second": second}
+
+
+def completed(sequence: int, activity_type: str, result: str) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "sequence": sequence,
+        "activity_type": activity_type,
+        "result": serializer.encode(result, codec="avro"),
+        "payload_codec": "avro",
+    }
+    if sequence == 1:
+        payload["reused_from_run_id"] = "failed-run"
+        payload["reused_activity_execution_id"] = "original-first"
+
+    return {
+        "event_type": "ActivityCompleted",
+        "payload": payload,
+    }
+
+
+def test_redriven_history_reuses_completed_prefix_and_retries_failed_step() -> None:
+    history = [completed(1, "tests.first", "recorded")]
+
+    retry = replay(RedriveWorkflow, history, []).commands
+    assert len(retry) == 1
+    assert isinstance(retry[0], ScheduleActivity)
+    assert retry[0].activity_type == "tests.second"
+    assert retry[0].arguments == ["recorded"]
+
+    completed_run = replay(
+        RedriveWorkflow,
+        history + [completed(2, "tests.second", "retried")],
+        [],
+    ).commands
+    assert len(completed_run) == 1
+    assert isinstance(completed_run[0], CompleteWorkflow)
+    assert completed_run[0].result == {"first": "recorded", "second": "retried"}

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_release_metadata.py

@@ -41,10 +41,10 @@ def test_worker_release_identity_matches_supported_server_and_protocol() -> None
     project = manifest["project"]
     release = manifest["tool"]["durable-workflow"]
 
-    assert project["version"] == "2.0.3"
+    assert project["version"] == "2.1.0"
     assert release["product-train"] == project["version"]
-    assert release["registry-version"] == "2.0.3"
-    assert release["supported-server-versions"] == "2.0.0"
+    assert release["registry-version"] == "2.1.0"
+    assert release["supported-server-versions"] == "2.4.0"
     assert release["worker-protocol-version"] == PROTOCOL_VERSION == "1.19"
     assert release["durable-selection"] is True
     assert release["durable-selection-minimum-worker-protocol-version"] == "1.19"
@@ -86,24 +86,30 @@ def test_release_metadata_loader_uses_tomli_without_stdlib_tomllib(
     assert imports == ["tomllib", "tomli"]
 
 
+@pytest.mark.parametrize(
+    ("version", "server_version"),
+    [("2.0.1", "2.0.0"), ("2.1.0", "2.4.0")],
+)
 def test_release_metadata_loader_accepts_the_authorized_stable_identity(
     monkeypatch: pytest.MonkeyPatch,
+    version: str,
+    server_version: str,
 ) -> None:
     commit = "b" * 40
-    pyproject = b"""
+    pyproject = f"""
 [project]
 name = "durable-workflow"
-version = "2.0.1"
+version = "{version}"
 description = "Python SDK for Durable Workflow 2.0"
 readme = "README.md"
 classifiers = ["Programming Language :: Python :: 3"]
 
 [tool.durable-workflow]
-product-train = "2.0.1"
-registry-version = "2.0.1"
-supported-server-versions = "2.0.0"
+product-train = "{version}"
+registry-version = "{version}"
+supported-server-versions = "{server_version}"
 worker-protocol-version = "1.19"
-"""
+""".encode()
     readme = b"# Durable Workflow\n\nBuild durable Python workflows.\n"
     runtime = b'PROTOCOL_VERSION = "1.19"\n'
 
@@ -120,9 +126,10 @@ worker-protocol-version = "1.19"
 
     monkeypatch.setattr(check_release_metadata, "_git", git)
 
-    source = check_release_metadata.load_source_metadata("2.0.1")
-    assert source.version == "2.0.1"
-    assert source.registry_version == "2.0.1"
+    source = check_release_metadata.load_source_metadata(version)
+    assert source.version == version
+    assert source.registry_version == version
+    assert source.server_version == server_version
 
 
 def test_normal_project_page_is_retained_as_rendered_evidence(monkeypatch: pytest.MonkeyPatch) -> None:

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_replay.py

@@ -60,6 +60,15 @@ class OneActivity:
         return {"greeting": result}
 
 
+@workflow.defn(name="translated-activity-failure")
+class TranslatedActivityFailure:
+    def run(self, ctx: WorkflowContext):  # type: ignore[no-untyped-def]
+        try:
+            yield ctx.schedule_activity("greet", [])
+        except ActivityFailed as exc:
+            raise RuntimeError("translated failure") from exc
+
+
 @workflow.defn(name="activity-failed-saga")
 class ActivityFailedSaga:
     def run(self, ctx: WorkflowContext, order_id: str):  # type: ignore[no-untyped-def]
@@ -442,6 +451,48 @@ class TestPublicReplayer:
 
 
 class TestOneActivity:
+    def test_uncaught_recorded_activity_failure_claims_only_its_persisted_boundary(self) -> None:
+        history = [{
+            "event_type": "ActivityFailed",
+            "payload": {
+                "sequence": 1,
+                "activity_type": "greet",
+                "activity_execution_id": "activity-1",
+                "message": "failed",
+            },
+        }]
+
+        outcome = replay(OneActivity, history, ["Ada"])
+        command = outcome.commands[0].to_server_command("workers")
+
+        assert command["type"] == "fail_workflow"
+        assert command["failed_step_sequence"] == 1
+        assert command["failed_activity_execution_id"] == "activity-1"
+
+        translated = replay(TranslatedActivityFailure, history, [])
+        translated_command = translated.commands[0].to_server_command("workers")
+        assert translated_command["type"] == "fail_workflow"
+        assert "failed_step_sequence" not in translated_command
+        assert "failed_activity_execution_id" not in translated_command
+
+        handled_history = [{
+            "event_type": "ActivityFailed",
+            "payload": {**history[0]["payload"], "activity_type": "charge-card"},
+        }]
+        handled = replay(ActivityFailedSaga, handled_history, ["order-1"])
+        assert isinstance(handled.commands[0], ScheduleActivity)
+
+    def test_activity_failure_without_complete_recorded_identity_does_not_claim_redrive(self) -> None:
+        for payload in (
+            {"sequence": 1, "activity_type": "greet", "message": "failed"},
+            {"activity_execution_id": "activity-1", "activity_type": "greet", "message": "failed"},
+        ):
+            outcome = replay(OneActivity, [{"event_type": "ActivityFailed", "payload": payload}], ["Ada"])
+            command = outcome.commands[0].to_server_command("workers")
+            assert command["type"] == "fail_workflow"
+            assert "failed_step_sequence" not in command
+            assert "failed_activity_execution_id" not in command
+
     def test_first_replay_schedules(self) -> None:
         outcome = replay(OneActivity, [], ["world"])
         assert len(outcome.commands) == 1

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_runtime_external_payload_transport.py

@@ -19,6 +19,7 @@ from durable_workflow.errors import (
     ExternalPayloadUnavailable,
     ExternalPayloadUnsupported,
     RuntimeCapabilityUnsupported,
+    ServerError,
 )
 from durable_workflow.external_storage import (
     RUNTIME_EXTERNAL_PAYLOAD_REFERENCE_SCHEMA,
@@ -174,6 +175,115 @@ class FakeRuntimePayloadServer:
         return httpx.Response(200, json=response)
 
 
+class CompletionPayloadServer(FakeRuntimePayloadServer):
+    def __init__(self, *, supported: bool = True, state: str = "draining", reject_bound: bool = False) -> None:
+        super().__init__()
+        self.supported = supported
+        self.state = state
+        self.reject_bound = reject_bound
+        self.upload_requests: list[httpx.Request] = []
+
+    def cluster_info(self) -> dict[str, Any]:
+        info = super().cluster_info()
+        if self.supported:
+            info["namespace"]["external_payload_storage"]["transport"]["upload"]["completion_context"] = {
+                "schema": "durable-workflow.v2.payload-completion-context.v1",
+                "header": "X-Durable-Workflow-Payload-Completion",
+            }
+        return info
+
+    def handler(self, request: httpx.Request) -> httpx.Response:
+        if request.method == "POST" and request.url.path == "/api/external-payloads/v1":
+            self.upload_requests.append(request)
+            if self.state != "normal" and "X-Durable-Workflow-Payload-Completion" not in request.headers:
+                return httpx.Response(503, json={"reason": "storage_pressure", "storage_state": self.state,
+                    "request_admitted": False, "retryable": True, "retry_after_seconds": 1})
+            if self.reject_bound:
+                return httpx.Response(409, json={"reason": "external_payload_completion_lease_rejected",
+                    "retryable": False, "message": "Lease rejected."})
+        return super().handler(request)
+
+
+def completion_cases() -> list[tuple[str, str, dict[str, Any], list[str | int]]]:
+    envelope = serializer.envelope("x" * 100)
+    activity = {"lease_owner": "worker", "activity_attempt_id": "attempt"}
+    cases = [
+        ("activity", "complete", {**activity, "result": envelope}, ["result"]),
+        ("activity", "fail", {**activity, "failure": {"details": envelope}}, ["failure", "details"]),
+        ("query", "complete", {"lease_owner": "worker", "query_task_attempt": 2, "result_envelope": envelope},
+            ["result_envelope"]),
+    ]
+    for kind, field in [("complete_workflow", "result"), ("schedule_activity", "arguments"),
+                        ("upsert_memo", "entries"), ("start_service_operation", "request_payload")]:
+        cases.append(("workflow", "complete", {"lease_owner": "worker", "workflow_task_attempt": 2,
+            "commands": [{"type": kind, field: envelope}]}, ["commands", 0, field]))
+    cases.append(("workflow", "complete", {"lease_owner": "worker", "workflow_task_attempt": 2,
+        "commands": [{"type": "fail_workflow", "exception": {"details": envelope}}]},
+        ["commands", 0, "exception", "details"]))
+    cases.append(("workflow", "complete", {"lease_owner": "worker", "workflow_task_attempt": 2,
+        "commands": [{"type": "record_side_effect", "result": serializer.encode("x" * 100)}]},
+        ["commands", 0, "result"]))
+    cases.append(("workflow", "complete", {"lease_owner": "worker", "workflow_task_attempt": 2,
+        "commands": [{"type": "record_side_effect", "workflow_stream": {"items": [
+            {"payload": envelope, "payload_codec": "avro"}]}}]},
+        ["commands", 0, "workflow_stream", "items", 0, "payload"]))
+    return cases
+
+
+@pytest.mark.parametrize("kind,operation,body,slot", completion_cases())
+async def test_draining_upload_retry_carries_current_completion_identity(
+    kind: str, operation: str, body: dict[str, Any], slot: list[str | int],
+) -> None:
+    server = CompletionPayloadServer()
+    async with runtime_client(server, retry_policy=TransportRetryPolicy(max_attempts=1)) as client:
+        await client._request("POST", f"/worker/{kind}-tasks/task/{operation}", worker=True, json=body)
+    first, bound = server.upload_requests
+    assert "X-Durable-Workflow-Payload-Completion" not in first.headers
+    assert json.loads(bound.headers["X-Durable-Workflow-Payload-Completion"]) == {
+        "schema": "durable-workflow.v2.payload-completion-context.v1", "kind": kind,
+        "task_id": "task", "attempt": "attempt" if kind == "activity" else 2,
+        "lease_owner": "worker", "operation": operation, "slot": slot,
+    }
+    assert first.content == bound.content
+    assert first.headers["authorization"] == bound.headers["authorization"]
+    assert first.headers["x-namespace"] == bound.headers["x-namespace"]
+    assert len(server.requests) == 1
+
+
+@pytest.mark.parametrize("supported,worker,state", [(False, True, "draining"), (True, False, "draining"),
+                                                    (True, True, "fenced")])
+async def test_completion_upload_does_not_bypass_unsupported_client_or_fenced_admission(
+    supported: bool, worker: bool, state: str,
+) -> None:
+    server = CompletionPayloadServer(supported=supported, state=state)
+    async with runtime_client(server, retry_policy=TransportRetryPolicy(max_attempts=1)) as client:
+        with pytest.raises((ServerError, ExternalPayloadError)):
+            await client._request("POST", "/worker/activity-tasks/task/complete" if worker else "/workflows",
+                worker=worker, json={"lease_owner": "worker", "activity_attempt_id": "attempt",
+                    "result" if worker else "input": serializer.envelope("x" * 100)})
+    assert len(server.upload_requests) == 1
+    assert server.requests == []
+
+
+async def test_rejected_bound_upload_does_not_loop_or_submit_completion() -> None:
+    server = CompletionPayloadServer(reject_bound=True)
+    async with runtime_client(server, retry_policy=TransportRetryPolicy(max_attempts=3)) as client:
+        with pytest.raises((ServerError, ExternalPayloadError)):
+            await client.complete_activity_task(task_id="task", activity_attempt_id="attempt", lease_owner="worker",
+                result="x" * 100)
+    assert len(server.upload_requests) == 2
+    assert server.requests == []
+
+
+async def test_normal_payload_upload_is_unchanged_when_completion_capability_exists() -> None:
+    server = CompletionPayloadServer(state="normal")
+    async with runtime_client(server) as client:
+        await client.complete_activity_task(task_id="task", activity_attempt_id="attempt", lease_owner="worker",
+            result="x" * 100)
+    assert len(server.upload_requests) == 1
+    assert "X-Durable-Workflow-Payload-Completion" not in server.upload_requests[0].headers
+
+
 def runtime_client(
     server: FakeRuntimePayloadServer,
     *,

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_storage_admission.py

@@ -225,8 +225,9 @@ async def test_shutdown_interrupts_pressure_without_another_mutation(monkeypatch
 
 
 @pytest.mark.parametrize("kind", ["activity", "query", "workflow"])
+@pytest.mark.parametrize("late_upload_refusal", [False, True])
 async def test_runtime_payload_upload_and_acknowledgement_are_not_repeated(
-    kind: str, retry_sleeps: list[float],
+    kind: str, retry_sleeps: list[float], late_upload_refusal: bool,
 ) -> None:
     server = FakeRuntimePayloadServer()
     uploads: list[bytes] = []
@@ -236,7 +237,10 @@ async def test_runtime_payload_upload_and_acknowledgement_are_not_repeated(
         if request.method == "POST" and request.url.path == "/api/external-payloads/v1":
             uploads.append(request.content)
             if len(uploads) <= 3:
-                return httpx.Response(503, json=pressure())
+                refusal = pressure()
+                if late_upload_refusal:
+                    refusal.pop("request_admitted")
+                return httpx.Response(503, json=refusal)
         if request.url.path.endswith("/complete"):
             acknowledgements.append(request.content)
             if len(acknowledgements) <= 4:

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_sync.py

@@ -519,6 +519,25 @@ class TestSyncClientRunVisibility:
 
 
 class TestSyncClientMaintenance:
+    def test_redrive_workflow(self) -> None:
+        client = Client("http://localhost:8080")
+        resp = _mock_response(
+            202,
+            {
+                "workflow_id": "wf-1",
+                "run_id": "run-2",
+                "outcome": "redriven",
+                "command_status": "accepted",
+            },
+        )
+        with patch.object(client._async._http, "request", new_callable=AsyncMock, return_value=resp) as mock:
+            result = client.redrive_workflow("wf-1", "run-1", request_id="retry-1")
+
+        assert result.run_id == "run-2"
+        assert result.outcome == "redriven"
+        assert mock.call_args.args[:2] == ("POST", "/api/workflows/wf-1/runs/run-1/redrive")
+        assert mock.call_args.kwargs["json"] == {"request_id": "retry-1"}
+
     def test_repair_workflow(self) -> None:
         client = Client("http://localhost:8080")
         resp = _mock_response(
@@ -786,6 +805,7 @@ class TestSyncWorkflowHandleControlPlane:
         async_handle.export_history = AsyncMock(return_value={"schema": "durable.workflow.history.v2"})  # type: ignore[method-assign]
         async_handle.list_runs = AsyncMock(return_value=[])  # type: ignore[method-assign]
         async_handle.describe_run = AsyncMock(return_value={"run_id": "r1"})  # type: ignore[method-assign]
+        async_handle.redrive = AsyncMock(return_value={"run_id": "r2"})  # type: ignore[method-assign]
         async_handle.repair = AsyncMock(return_value={"outcome": "accepted"})  # type: ignore[method-assign]
         async_handle.archive = AsyncMock(return_value={"outcome": "completed"})  # type: ignore[method-assign]
         with Client("http://localhost:8080") as client:
@@ -796,7 +816,9 @@ class TestSyncWorkflowHandleControlPlane:
             assert handle.list_runs() == []
             assert handle.describe_run() == {"run_id": "r1"}
             assert handle.repair() == {"outcome": "accepted"}
+            assert handle.redrive(request_id="retry-1") == {"run_id": "r2"}
             assert handle.archive(reason="retention") == {"outcome": "completed"}
+            async_handle.redrive.assert_awaited_once_with(request_id="retry-1")
             async_handle.archive.assert_awaited_once_with(reason="retention")
 
 

{durable_workflow-2.0.3 → durable_workflow-2.1.0}/tests/test_worker.py

@@ -1123,6 +1123,33 @@ class TestWorkflowTaskExecution:
         assert commands[0]["arguments"]["codec"] == "avro"
         assert serializer.decode(commands[0]["arguments"]["blob"], codec="avro") == ["hello"]
 
+    @pytest.mark.asyncio
+    async def test_uncaught_recorded_activity_failure_sends_redrive_boundary(
+        self, mock_client: AsyncMock
+    ) -> None:
+        worker = Worker(mock_client, task_queue="q1", workflows=[TestWorkflow], activities=[])
+        await worker._run_workflow_task({
+            "task_id": "failed-activity-task",
+            "workflow_type": "test-wf",
+            "workflow_task_attempt": 1,
+            "history_events": [{
+                "event_type": "ActivityFailed",
+                "payload": {
+                    "sequence": 1,
+                    "activity_type": "test-act",
+                    "activity_execution_id": "activity-execution-1",
+                    "message": "failed",
+                },
+            }],
+            "arguments": serializer.encode(["hello"], codec="avro"),
+            "payload_codec": "avro",
+        })
+
+        commands = mock_client.complete_workflow_task.await_args.kwargs["commands"]
+        assert commands[0]["type"] == "fail_workflow"
+        assert commands[0]["failed_step_sequence"] == 1
+        assert commands[0]["failed_activity_execution_id"] == "activity-execution-1"
+
     @pytest.mark.asyncio
     async def test_workflow_task_ambiguous_completion_error_preserves_commands(
         self, mock_client: AsyncMock