durable-workflow 2.3.1__tar.gz → 2.3.4__tar.gz

{durable_workflow-2.3.1/src/durable_workflow.egg-info → durable_workflow-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.1
+Version: 2.3.4
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.3.1"
+version = "2.3.4"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -71,8 +71,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.3.1"
-registry-version = "2.3.1"
+product-train = "2.3.4"
+registry-version = "2.3.4"
 supported-server-versions = "2.4.8"
 worker-protocol-version = "1.19"
 control-plane-version = "2"

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/src/durable_workflow/errors.py

@@ -171,6 +171,25 @@ class ServerError(DurableWorkflowError):
             return self.body.get("reason")
         return None
 
+    def poll_capacity_backpressure_delay(self, task_kind: str, task_queue: str) -> int | None:
+        """Return the advertised wait only for a complete poll-capacity refusal."""
+        body = self.body
+        if (
+            self.status != 429
+            or not isinstance(body, dict)
+            or body.get("reason") != "long_poll_capacity_exhausted"
+            or body.get("poll_status") != "long_poll_capacity_exhausted"
+            or "task" not in body or body["task"] is not None
+            or body.get("task_kind") != task_kind
+            or body.get("task_queue") != task_queue
+            or body.get("retryable") is not True
+            or type(body.get("retry_after_seconds")) is not int
+            or body["retry_after_seconds"] <= 0
+            or body["retry_after_seconds"] > 60
+        ):
+            return None
+        return int(body["retry_after_seconds"])
+
     def is_storage_admission_failure(self, poll_request_id: str | None = None) -> bool:
         """Whether the runtime explicitly refused admission and requested an identity-preserving retry."""
         body = self.body

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/src/durable_workflow/retry_policy.py

@@ -80,6 +80,18 @@ def _backend_unavailable_refusal(exc: Exception) -> tuple[bool, int | None]:
         "/api/worker/heartbeat": "heartbeat_worker",
     }
     operation = next((name for path, name in operations.items() if request.url.path.endswith(path)), None)
+    _, task_heartbeat_marker, task_heartbeat_tail = request.url.path.rpartition(
+        "/api/worker/workflow-tasks/"
+    )
+    task_id = task_heartbeat_tail.removesuffix("/heartbeat")
+    task_heartbeat = (
+        bool(task_heartbeat_marker)
+        and task_heartbeat_tail.endswith("/heartbeat")
+        and bool(task_id)
+        and "/" not in task_id
+    )
+    if task_heartbeat:
+        operation = "heartbeat_workflow_task"
     if operation is None:
         return False, None
     try:
@@ -97,6 +109,24 @@ def _backend_unavailable_refusal(exc: Exception) -> tuple[bool, int | None]:
     worker_id = submitted.get("worker_id")
     queue = submitted.get("task_queue")
     delay = body.get("retry_after_seconds")
+    if task_heartbeat:
+        lease_owner = submitted.get("lease_owner")
+        attempt = submitted.get("workflow_task_attempt")
+        if (
+            not isinstance(lease_owner, str) or not lease_owner
+            or type(attempt) is not int or attempt <= 0
+            or body.get("operation") != operation
+            or body.get("outcome") != "unknown"
+            or body.get("worker_id") != lease_owner
+            or body.get("task_queue") is not None
+            or body.get("task_id") != task_id
+            or body.get("lease_owner") != lease_owner
+            or body.get("workflow_task_attempt") != attempt
+            or body.get("retryable") is not True
+            or type(delay) is not int or delay <= 0
+        ):
+            return True, None
+        return True, delay
     if (
         not isinstance(worker_id, str) or not worker_id
         or body.get("operation") != operation
@@ -121,6 +151,32 @@ def _backend_unavailable_refusal(exc: Exception) -> tuple[bool, int | None]:
     return True, delay
 
 
+def _poll_capacity_refusal(exc: Exception) -> bool:
+    if not isinstance(exc, httpx.HTTPStatusError) or exc.response.status_code != 429:
+        return False
+    request = exc.request
+    if request.method != "POST" or "X-Durable-Workflow-Protocol-Version" not in request.headers:
+        return False
+    task_kinds = {
+        "/api/worker/workflow-tasks/poll": "workflow_task",
+        "/api/worker/activity-tasks/poll": "activity_task",
+        "/api/worker/query-tasks/poll": "query_task",
+    }
+    task_kind = next((kind for path, kind in task_kinds.items() if request.url.path.endswith(path)), None)
+    if task_kind is None:
+        return False
+    try:
+        body = exc.response.json()
+        submitted = json.loads(request.content)
+    except ValueError:
+        return False
+    if not isinstance(submitted, dict) or not isinstance(submitted.get("task_queue"), str):
+        return False
+    return ServerError(429, body).poll_capacity_backpressure_delay(
+        task_kind, submitted["task_queue"],
+    ) is not None
+
+
 @dataclass
 class TransportRetryPolicy:
     """
@@ -157,6 +213,8 @@ class TransportRetryPolicy:
 
         # Retry 5xx server errors and 429 rate limit
         if isinstance(exc, httpx.HTTPStatusError):
+            if _poll_capacity_refusal(exc):
+                return False
             return exc.response.status_code >= 500 or exc.response.status_code == 429
 
         return False

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/src/durable_workflow/worker.py

@@ -190,6 +190,12 @@ def _poll_error_summary(error: BaseException) -> str:
     return f"server returned {error.status} reason={reason[:64]!r}"
 
 
+def _poll_capacity_delay(error: BaseException, task_kind: str, task_queue: str) -> int | None:
+    if not isinstance(error, ServerError):
+        return None
+    return error.poll_capacity_backpressure_delay(task_kind, task_queue)
+
+
 def _command_payload_codec(codec: object) -> str:
     if codec is None:
         return serializer.AVRO_CODEC
@@ -2539,6 +2545,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "workflow_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("workflow", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("workflow poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("workflow", "error", time.perf_counter() - poll_start)
                 log.warning("workflow poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2637,6 +2649,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "activity_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("activity", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("activity poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("activity", "error", time.perf_counter() - poll_start)
                 log.warning("activity poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2688,6 +2706,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "query_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("query", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("query poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("query", "error", time.perf_counter() - poll_start)
                 log.warning("query poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)

{durable_workflow-2.3.1 → durable_workflow-2.3.4/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.1
+Version: 2.3.4
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/tests/test_client.py

@@ -2682,6 +2682,34 @@ class TestFailWorkflowTask:
         assert isinstance(request_body["poll_request_id"], str)
         assert request_body["poll_request_id"] != ""
 
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_reaches_worker_after_one_http_request(
+        self, client: Client,
+    ) -> None:
+        requests: list[httpx.Request] = []
+
+        async def refused(method: str, path: str, **kwargs: object) -> httpx.Response:
+            request = httpx.Request(
+                method, "http://localhost:8080" + path,
+                headers=kwargs["headers"], json=kwargs["json"],
+            )
+            requests.append(request)
+            return httpx.Response(429, request=request, json={
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": "workflow_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            })
+
+        with patch.object(client._http, "request", side_effect=refused), pytest.raises(ServerError) as refusal:
+            await client.poll_workflow_task(worker_id="worker-1", task_queue="q1")
+
+        assert len(requests) == 1
+        assert refusal.value.poll_capacity_backpressure_delay("workflow_task", "q1") == 2
+
     @pytest.mark.asyncio
     async def test_poll_workflow_task_response_preserves_no_compatible_status(self, client: Client) -> None:
         response = {

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/tests/test_retry_policy.py

@@ -40,6 +40,60 @@ class TestRetryPolicy:
         exc = httpx.HTTPStatusError("rate limited", request=response.request, response=response)
         assert policy.should_retry(exc, attempt=0) is True
 
+    @pytest.mark.parametrize(
+        ("path", "task_kind"),
+        [
+            ("/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/runtime/v1/namespaces/acme/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/worker/activity-tasks/poll", "activity_task"),
+            ("/api/worker/query-tasks/poll", "query_task"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_does_not_repeat_transport_request(
+        self, path: str, task_kind: str,
+    ) -> None:
+        policy = RetryPolicy(max_attempts=3, jitter=False)
+        request = httpx.Request(
+            "POST", "http://test" + path,
+            headers={"X-Durable-Workflow-Protocol-Version": "1.19"},
+            json={"task_queue": "q1", "worker_id": "worker-1"},
+        )
+        response = httpx.Response(429, request=request, json={
+            "task": None,
+            "poll_status": "long_poll_capacity_exhausted",
+            "reason": "long_poll_capacity_exhausted",
+            "task_kind": task_kind,
+            "task_queue": "q1",
+            "retryable": True,
+            "retry_after_seconds": 2,
+        })
+        error = httpx.HTTPStatusError("capacity", request=request, response=response)
+        calls = 0
+
+        async def refused() -> None:
+            nonlocal calls
+            calls += 1
+            raise error
+
+        with pytest.raises(httpx.HTTPStatusError):
+            await policy.execute(refused)
+        assert calls == 1
+
+        wrong_kind = httpx.Response(429, request=request, json={
+            **response.json(), "task_kind": task_kind.removesuffix("_task"),
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("wrong kind", request=request, response=wrong_kind), attempt=0,
+        ) is True
+
+        malformed = httpx.Response(429, request=request, json={
+            "reason": "long_poll_capacity_exhausted", "retry_after_seconds": 2,
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("malformed", request=request, response=malformed), attempt=0,
+        ) is True
+
     def test_should_not_retry_4xx_client_error(self) -> None:
         policy = RetryPolicy(max_attempts=3)
         response = httpx.Response(status_code=404, request=httpx.Request("GET", "http://test"))

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/tests/test_storage_admission.py

@@ -42,7 +42,8 @@ def pressure(poll_id: str | None = None, *, reason: str = "storage_pressure", en
 def backend_unavailable(request: httpx.Request) -> dict[str, Any]:
     submitted = json.loads(request.content)
     path = request.url.path
-    operation = {
+    task_heartbeat = "/api/worker/workflow-tasks/" in path and path.endswith("/heartbeat")
+    operation = "heartbeat_workflow_task" if task_heartbeat else {
         "/api/worker/workflow-tasks/poll": "poll_workflow_task",
         "/api/worker/activity-tasks/poll": "poll_activity_task",
         "/api/worker/query-tasks/poll": "poll_query_task",
@@ -51,9 +52,16 @@ def backend_unavailable(request: httpx.Request) -> dict[str, Any]:
     }[path]
     body: dict[str, Any] = {
         "reason": "backend_unavailable", "operation": operation, "outcome": "unknown",
-        "worker_id": submitted["worker_id"], "task_queue": submitted.get("task_queue"),
+        "worker_id": submitted.get("lease_owner") if task_heartbeat else submitted["worker_id"],
+        "task_queue": submitted.get("task_queue"),
         "retryable": True, "retry_after_seconds": 1,
     }
+    if task_heartbeat:
+        body.update({
+            "task_id": path.rpartition("/api/worker/workflow-tasks/")[2].removesuffix("/heartbeat"),
+            "lease_owner": submitted["lease_owner"],
+            "workflow_task_attempt": submitted["workflow_task_attempt"],
+        })
     if path.endswith("/poll"):
         body.update({
             "task": None, "poll_status": "backend_unavailable",
@@ -85,9 +93,9 @@ def retry_sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]:
     return sleeps
 
 
-def client_for(handler: Callable[..., Any]) -> Client:
+def client_for(handler: Callable[..., Any], base_url: str = "https://runtime.example") -> Client:
     client = Client(
-        "https://runtime.example", token="test-runtime-token",
+        base_url, token="test-runtime-token",
         retry_policy=TransportRetryPolicy(max_attempts=2, initial_backoff_seconds=0, jitter=False),
     )
     client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(handler))
@@ -186,6 +194,57 @@ async def test_backend_outage_retries_worker_registration_and_heartbeat(
     assert sum(retry_sleeps) == pytest.approx(4)
 
 
+@pytest.mark.parametrize("base_url", ["https://runtime.example", "https://runtime.example/managed/namespace"])
+async def test_backend_outage_retries_workflow_task_heartbeat_with_same_fence(
+    base_url: str, retry_sleeps: list[float],
+) -> None:
+    requests: list[bytes] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        requests.append(request.content)
+        if len(requests) <= 4:
+            return httpx.Response(503, json=backend_unavailable(request))
+        return httpx.Response(200, json={
+            "task_id": "task-1", "lease_owner": "worker-1",
+            "workflow_task_attempt": 3, "renewed": True,
+        })
+
+    async with client_for(handler, base_url) as client:
+        with worker_scope():
+            result = await client.heartbeat_workflow_task(
+                task_id="task-1", lease_owner="worker-1", workflow_task_attempt=3,
+            )
+    assert result["renewed"] is True
+    assert len(requests) == 5
+    assert len(set(requests)) == 1
+    assert sum(retry_sleeps) == pytest.approx(4)
+
+
+@pytest.mark.parametrize("override", [
+    {"task_id": "other-task"}, {"lease_owner": "other-worker"},
+    {"worker_id": "other-worker"}, {"workflow_task_attempt": 4},
+    {"operation": "heartbeat_worker"}, {"outcome": "failed"},
+    {"retryable": False}, {"retry_after_seconds": 0},
+])
+async def test_invalid_workflow_task_heartbeat_backend_response_is_not_retried(
+    override: dict[str, Any], retry_sleeps: list[float],
+) -> None:
+    calls = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json={**backend_unavailable(request), **override})
+
+    async with client_for(handler) as client:
+        with worker_scope(), pytest.raises(ServerError):
+            await client.heartbeat_workflow_task(
+                task_id="task-1", lease_owner="worker-1", workflow_task_attempt=3,
+            )
+    assert calls == 1
+    assert not retry_sleeps
+
+
 @pytest.mark.parametrize("override", [
     {"operation": "poll_activity_task"},
     {"outcome": "failed"}, {"worker_id": "other-worker"},

{durable_workflow-2.3.1 → durable_workflow-2.3.4}/tests/test_worker.py

@@ -867,7 +867,60 @@ class TestWorkerRegistration:
         ],
     )
     @pytest.mark.asyncio
-    async def test_poll_warning_omits_large_server_response(
+    async def test_typed_poll_backpressure_waits_without_warning(
+        self,
+        mock_client: AsyncMock,
+        monkeypatch: pytest.MonkeyPatch,
+        caplog: pytest.LogCaptureFixture,
+        poll_method: str,
+        loop_method: str,
+        task_kind: str,
+    ) -> None:
+        worker = Worker(
+            mock_client,
+            task_queue="q1",
+            workflows=[TestWorkflow],
+            activities=[echo_activity],
+        )
+        getattr(mock_client, poll_method).side_effect = ServerError(
+            429,
+            {
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": f"{task_kind}_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            },
+        )
+        outcomes: list[tuple[str, str]] = []
+        monkeypatch.setattr(
+            worker, "_record_poll_metrics",
+            lambda kind, outcome, _duration: outcomes.append((kind, outcome)),
+        )
+
+        async def stop_after_backoff(seconds: float) -> None:
+            assert seconds == 2
+            worker._stop.set()
+
+        monkeypatch.setattr(worker_module.asyncio, "sleep", stop_after_backoff)
+        with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"):
+            await getattr(worker, loop_method)()
+
+        assert outcomes == [(task_kind, "backpressure")]
+        assert not [record for record in caplog.records if record.name == "durable_workflow.worker"]
+
+    @pytest.mark.parametrize(
+        ("poll_method", "loop_method", "task_kind"),
+        [
+            ("poll_workflow_task", "_poll_workflow_tasks", "workflow"),
+            ("poll_activity_task", "_poll_activity_tasks", "activity"),
+            ("poll_query_task", "_poll_query_tasks", "query"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_malformed_poll_warning_omits_large_server_response(
         self,
         mock_client: AsyncMock,
         monkeypatch: pytest.MonkeyPatch,