durable-workflow 2.3.1__tar.gz → 2.3.3__tar.gz

{durable_workflow-2.3.1/src/durable_workflow.egg-info → durable_workflow-2.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.1
+Version: 2.3.3
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.3.1"
+version = "2.3.3"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -71,8 +71,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.3.1"
-registry-version = "2.3.1"
+product-train = "2.3.3"
+registry-version = "2.3.3"
 supported-server-versions = "2.4.8"
 worker-protocol-version = "1.19"
 control-plane-version = "2"

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/src/durable_workflow/errors.py

@@ -171,6 +171,25 @@ class ServerError(DurableWorkflowError):
             return self.body.get("reason")
         return None
 
+    def poll_capacity_backpressure_delay(self, task_kind: str, task_queue: str) -> int | None:
+        """Return the advertised wait only for a complete poll-capacity refusal."""
+        body = self.body
+        if (
+            self.status != 429
+            or not isinstance(body, dict)
+            or body.get("reason") != "long_poll_capacity_exhausted"
+            or body.get("poll_status") != "long_poll_capacity_exhausted"
+            or "task" not in body or body["task"] is not None
+            or body.get("task_kind") != task_kind
+            or body.get("task_queue") != task_queue
+            or body.get("retryable") is not True
+            or type(body.get("retry_after_seconds")) is not int
+            or body["retry_after_seconds"] <= 0
+            or body["retry_after_seconds"] > 60
+        ):
+            return None
+        return int(body["retry_after_seconds"])
+
     def is_storage_admission_failure(self, poll_request_id: str | None = None) -> bool:
         """Whether the runtime explicitly refused admission and requested an identity-preserving retry."""
         body = self.body

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/src/durable_workflow/retry_policy.py

@@ -121,6 +121,32 @@ def _backend_unavailable_refusal(exc: Exception) -> tuple[bool, int | None]:
     return True, delay
 
 
+def _poll_capacity_refusal(exc: Exception) -> bool:
+    if not isinstance(exc, httpx.HTTPStatusError) or exc.response.status_code != 429:
+        return False
+    request = exc.request
+    if request.method != "POST" or "X-Durable-Workflow-Protocol-Version" not in request.headers:
+        return False
+    task_kinds = {
+        "/api/worker/workflow-tasks/poll": "workflow_task",
+        "/api/worker/activity-tasks/poll": "activity_task",
+        "/api/worker/query-tasks/poll": "query_task",
+    }
+    task_kind = next((kind for path, kind in task_kinds.items() if request.url.path.endswith(path)), None)
+    if task_kind is None:
+        return False
+    try:
+        body = exc.response.json()
+        submitted = json.loads(request.content)
+    except ValueError:
+        return False
+    if not isinstance(submitted, dict) or not isinstance(submitted.get("task_queue"), str):
+        return False
+    return ServerError(429, body).poll_capacity_backpressure_delay(
+        task_kind, submitted["task_queue"],
+    ) is not None
+
+
 @dataclass
 class TransportRetryPolicy:
     """
@@ -157,6 +183,8 @@ class TransportRetryPolicy:
 
         # Retry 5xx server errors and 429 rate limit
         if isinstance(exc, httpx.HTTPStatusError):
+            if _poll_capacity_refusal(exc):
+                return False
             return exc.response.status_code >= 500 or exc.response.status_code == 429
 
         return False

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/src/durable_workflow/worker.py

@@ -190,6 +190,12 @@ def _poll_error_summary(error: BaseException) -> str:
     return f"server returned {error.status} reason={reason[:64]!r}"
 
 
+def _poll_capacity_delay(error: BaseException, task_kind: str, task_queue: str) -> int | None:
+    if not isinstance(error, ServerError):
+        return None
+    return error.poll_capacity_backpressure_delay(task_kind, task_queue)
+
+
 def _command_payload_codec(codec: object) -> str:
     if codec is None:
         return serializer.AVRO_CODEC
@@ -2539,6 +2545,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "workflow_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("workflow", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("workflow poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("workflow", "error", time.perf_counter() - poll_start)
                 log.warning("workflow poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2637,6 +2649,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "activity_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("activity", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("activity poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("activity", "error", time.perf_counter() - poll_start)
                 log.warning("activity poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2688,6 +2706,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "query_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("query", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("query poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("query", "error", time.perf_counter() - poll_start)
                 log.warning("query poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)

{durable_workflow-2.3.1 → durable_workflow-2.3.3/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.1
+Version: 2.3.3
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/tests/test_client.py

@@ -2682,6 +2682,34 @@ class TestFailWorkflowTask:
         assert isinstance(request_body["poll_request_id"], str)
         assert request_body["poll_request_id"] != ""
 
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_reaches_worker_after_one_http_request(
+        self, client: Client,
+    ) -> None:
+        requests: list[httpx.Request] = []
+
+        async def refused(method: str, path: str, **kwargs: object) -> httpx.Response:
+            request = httpx.Request(
+                method, "http://localhost:8080" + path,
+                headers=kwargs["headers"], json=kwargs["json"],
+            )
+            requests.append(request)
+            return httpx.Response(429, request=request, json={
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": "workflow_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            })
+
+        with patch.object(client._http, "request", side_effect=refused), pytest.raises(ServerError) as refusal:
+            await client.poll_workflow_task(worker_id="worker-1", task_queue="q1")
+
+        assert len(requests) == 1
+        assert refusal.value.poll_capacity_backpressure_delay("workflow_task", "q1") == 2
+
     @pytest.mark.asyncio
     async def test_poll_workflow_task_response_preserves_no_compatible_status(self, client: Client) -> None:
         response = {

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/tests/test_retry_policy.py

@@ -40,6 +40,60 @@ class TestRetryPolicy:
         exc = httpx.HTTPStatusError("rate limited", request=response.request, response=response)
         assert policy.should_retry(exc, attempt=0) is True
 
+    @pytest.mark.parametrize(
+        ("path", "task_kind"),
+        [
+            ("/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/runtime/v1/namespaces/acme/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/worker/activity-tasks/poll", "activity_task"),
+            ("/api/worker/query-tasks/poll", "query_task"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_does_not_repeat_transport_request(
+        self, path: str, task_kind: str,
+    ) -> None:
+        policy = RetryPolicy(max_attempts=3, jitter=False)
+        request = httpx.Request(
+            "POST", "http://test" + path,
+            headers={"X-Durable-Workflow-Protocol-Version": "1.19"},
+            json={"task_queue": "q1", "worker_id": "worker-1"},
+        )
+        response = httpx.Response(429, request=request, json={
+            "task": None,
+            "poll_status": "long_poll_capacity_exhausted",
+            "reason": "long_poll_capacity_exhausted",
+            "task_kind": task_kind,
+            "task_queue": "q1",
+            "retryable": True,
+            "retry_after_seconds": 2,
+        })
+        error = httpx.HTTPStatusError("capacity", request=request, response=response)
+        calls = 0
+
+        async def refused() -> None:
+            nonlocal calls
+            calls += 1
+            raise error
+
+        with pytest.raises(httpx.HTTPStatusError):
+            await policy.execute(refused)
+        assert calls == 1
+
+        wrong_kind = httpx.Response(429, request=request, json={
+            **response.json(), "task_kind": task_kind.removesuffix("_task"),
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("wrong kind", request=request, response=wrong_kind), attempt=0,
+        ) is True
+
+        malformed = httpx.Response(429, request=request, json={
+            "reason": "long_poll_capacity_exhausted", "retry_after_seconds": 2,
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("malformed", request=request, response=malformed), attempt=0,
+        ) is True
+
     def test_should_not_retry_4xx_client_error(self) -> None:
         policy = RetryPolicy(max_attempts=3)
         response = httpx.Response(status_code=404, request=httpx.Request("GET", "http://test"))

{durable_workflow-2.3.1 → durable_workflow-2.3.3}/tests/test_worker.py

@@ -867,7 +867,60 @@ class TestWorkerRegistration:
         ],
     )
     @pytest.mark.asyncio
-    async def test_poll_warning_omits_large_server_response(
+    async def test_typed_poll_backpressure_waits_without_warning(
+        self,
+        mock_client: AsyncMock,
+        monkeypatch: pytest.MonkeyPatch,
+        caplog: pytest.LogCaptureFixture,
+        poll_method: str,
+        loop_method: str,
+        task_kind: str,
+    ) -> None:
+        worker = Worker(
+            mock_client,
+            task_queue="q1",
+            workflows=[TestWorkflow],
+            activities=[echo_activity],
+        )
+        getattr(mock_client, poll_method).side_effect = ServerError(
+            429,
+            {
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": f"{task_kind}_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            },
+        )
+        outcomes: list[tuple[str, str]] = []
+        monkeypatch.setattr(
+            worker, "_record_poll_metrics",
+            lambda kind, outcome, _duration: outcomes.append((kind, outcome)),
+        )
+
+        async def stop_after_backoff(seconds: float) -> None:
+            assert seconds == 2
+            worker._stop.set()
+
+        monkeypatch.setattr(worker_module.asyncio, "sleep", stop_after_backoff)
+        with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"):
+            await getattr(worker, loop_method)()
+
+        assert outcomes == [(task_kind, "backpressure")]
+        assert not [record for record in caplog.records if record.name == "durable_workflow.worker"]
+
+    @pytest.mark.parametrize(
+        ("poll_method", "loop_method", "task_kind"),
+        [
+            ("poll_workflow_task", "_poll_workflow_tasks", "workflow"),
+            ("poll_activity_task", "_poll_activity_tasks", "activity"),
+            ("poll_query_task", "_poll_query_tasks", "query"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_malformed_poll_warning_omits_large_server_response(
         self,
         mock_client: AsyncMock,
         monkeypatch: pytest.MonkeyPatch,