durable-workflow 2.3.0__tar.gz → 2.3.3__tar.gz

{durable_workflow-2.3.0/src/durable_workflow.egg-info → durable_workflow-2.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.0
+Version: 2.3.3
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.3.0"
+version = "2.3.3"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -71,9 +71,9 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.3.0"
-registry-version = "2.3.0"
-supported-server-versions = "2.4.2"
+product-train = "2.3.3"
+registry-version = "2.3.3"
+supported-server-versions = "2.4.8"
 worker-protocol-version = "1.19"
 control-plane-version = "2"
 durable-selection = true

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/src/durable_workflow/errors.py

@@ -171,6 +171,25 @@ class ServerError(DurableWorkflowError):
             return self.body.get("reason")
         return None
 
+    def poll_capacity_backpressure_delay(self, task_kind: str, task_queue: str) -> int | None:
+        """Return the advertised wait only for a complete poll-capacity refusal."""
+        body = self.body
+        if (
+            self.status != 429
+            or not isinstance(body, dict)
+            or body.get("reason") != "long_poll_capacity_exhausted"
+            or body.get("poll_status") != "long_poll_capacity_exhausted"
+            or "task" not in body or body["task"] is not None
+            or body.get("task_kind") != task_kind
+            or body.get("task_queue") != task_queue
+            or body.get("retryable") is not True
+            or type(body.get("retry_after_seconds")) is not int
+            or body["retry_after_seconds"] <= 0
+            or body["retry_after_seconds"] > 60
+        ):
+            return None
+        return int(body["retry_after_seconds"])
+
     def is_storage_admission_failure(self, poll_request_id: str | None = None) -> bool:
         """Whether the runtime explicitly refused admission and requested an identity-preserving retry."""
         body = self.body

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/src/durable_workflow/retry_policy.py

@@ -65,6 +65,88 @@ def _storage_refusal(exc: Exception) -> tuple[ServerError, str | None] | None:
     return error, poll_id
 
 
+def _backend_unavailable_refusal(exc: Exception) -> tuple[bool, int | None]:
+    if not isinstance(exc, httpx.HTTPStatusError) or exc.response.status_code != 503:
+        return False, None
+    request = exc.request
+    if request.method != "POST" or "X-Durable-Workflow-Protocol-Version" not in request.headers:
+        return False, None
+    operations = {
+        "/api/worker/workflow-tasks/poll": "poll_workflow_task",
+        "/api/worker/activity-tasks/poll": "poll_activity_task",
+        "/api/worker/query-tasks/poll": "poll_query_task",
+        "/api/worker/update-validation-tasks/poll": "poll_update_validation_task",
+        "/api/worker/register": "register_worker",
+        "/api/worker/heartbeat": "heartbeat_worker",
+    }
+    operation = next((name for path, name in operations.items() if request.url.path.endswith(path)), None)
+    if operation is None:
+        return False, None
+    try:
+        body = exc.response.json()
+    except ValueError:
+        return False, None
+    if not isinstance(body, dict) or body.get("reason") != "backend_unavailable":
+        return False, None
+    try:
+        submitted = json.loads(request.content)
+    except ValueError:
+        return True, None
+    if not isinstance(submitted, dict):
+        return True, None
+    worker_id = submitted.get("worker_id")
+    queue = submitted.get("task_queue")
+    delay = body.get("retry_after_seconds")
+    if (
+        not isinstance(worker_id, str) or not worker_id
+        or body.get("operation") != operation
+        or body.get("outcome") != "unknown"
+        or body.get("worker_id") != worker_id
+        or body.get("task_queue") != queue
+        or body.get("retryable") is not True
+        or type(delay) is not int or delay <= 0
+        or (operation != "heartbeat_worker" and (not isinstance(queue, str) or not queue))
+    ):
+        return True, None
+    if operation.startswith("poll_"):
+        poll_id = submitted.get("poll_request_id")
+        if (
+            not isinstance(poll_id, str) or not poll_id
+            or "task" not in body or body["task"] is not None
+            or body.get("poll_status") != "backend_unavailable"
+            or body.get("poll_request_id") != poll_id
+            or body.get("retry_same_poll_request_id") is not True
+        ):
+            return True, None
+    return True, delay
+
+
+def _poll_capacity_refusal(exc: Exception) -> bool:
+    if not isinstance(exc, httpx.HTTPStatusError) or exc.response.status_code != 429:
+        return False
+    request = exc.request
+    if request.method != "POST" or "X-Durable-Workflow-Protocol-Version" not in request.headers:
+        return False
+    task_kinds = {
+        "/api/worker/workflow-tasks/poll": "workflow_task",
+        "/api/worker/activity-tasks/poll": "activity_task",
+        "/api/worker/query-tasks/poll": "query_task",
+    }
+    task_kind = next((kind for path, kind in task_kinds.items() if request.url.path.endswith(path)), None)
+    if task_kind is None:
+        return False
+    try:
+        body = exc.response.json()
+        submitted = json.loads(request.content)
+    except ValueError:
+        return False
+    if not isinstance(submitted, dict) or not isinstance(submitted.get("task_queue"), str):
+        return False
+    return ServerError(429, body).poll_capacity_backpressure_delay(
+        task_kind, submitted["task_queue"],
+    ) is not None
+
+
 @dataclass
 class TransportRetryPolicy:
     """
@@ -101,6 +183,8 @@ class TransportRetryPolicy:
 
         # Retry 5xx server errors and 429 rate limit
         if isinstance(exc, httpx.HTTPStatusError):
+            if _poll_capacity_refusal(exc):
+                return False
             return exc.response.status_code >= 500 or exc.response.status_code == 429
 
         return False
@@ -123,7 +207,7 @@ class TransportRetryPolicy:
         Raises the last exception if all retries are exhausted.
         """
         attempt = 0
-        storage_attempt = 0
+        worker_pause_attempt = 0
         last_exc: Exception | None = None
 
         while attempt < self.max_attempts:
@@ -134,20 +218,30 @@ class TransportRetryPolicy:
                 last_exc = exc
                 stop = _worker_storage_admission_stop.get()
                 refusal = _storage_refusal(exc) if stop is not None else None
-                if refusal is not None and stop is not None:
+                backend_refusal, backend_delay = _backend_unavailable_refusal(exc)
+                if backend_refusal and backend_delay is None:
+                    raise
+                pause: tuple[str, int] | None = None
+                if refusal is not None:
                     error, poll_id = refusal
-                    if not error.is_storage_admission_failure(poll_id) or stop():
+                    if not error.is_storage_admission_failure(poll_id):
                         raise
-                    storage_attempt += 1
                     assert isinstance(error.body, dict)
+                    pause = ("storage admission paused", error.body["retry_after_seconds"])
+                elif backend_delay is not None:
+                    pause = ("worker backend unavailable", backend_delay)
+                if pause is not None and stop is not None:
+                    if stop():
+                        raise
+                    worker_pause_attempt += 1
                     delay = min(
                         5.0,
                         max(
-                            self.backoff_seconds(min(storage_attempt - 1, 6)),
-                            error.body["retry_after_seconds"],
+                            self.backoff_seconds(min(worker_pause_attempt - 1, 6)),
+                            pause[1],
                         ),
                     )
-                    log.warning("storage admission paused; retrying the same worker request in %.2fs", delay)
+                    log.warning("%s; retrying the same worker request in %.2fs", pause[0], delay)
                     # Do not consume the finite transport budget or repeat serialization/uploads.
                     while delay > 0:
                         if stop():

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/src/durable_workflow/worker.py

@@ -190,6 +190,12 @@ def _poll_error_summary(error: BaseException) -> str:
     return f"server returned {error.status} reason={reason[:64]!r}"
 
 
+def _poll_capacity_delay(error: BaseException, task_kind: str, task_queue: str) -> int | None:
+    if not isinstance(error, ServerError):
+        return None
+    return error.poll_capacity_backpressure_delay(task_kind, task_queue)
+
+
 def _command_payload_codec(codec: object) -> str:
     if codec is None:
         return serializer.AVRO_CODEC
@@ -2539,6 +2545,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "workflow_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("workflow", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("workflow poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("workflow", "error", time.perf_counter() - poll_start)
                 log.warning("workflow poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2637,6 +2649,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "activity_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("activity", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("activity poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("activity", "error", time.perf_counter() - poll_start)
                 log.warning("activity poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)
@@ -2688,6 +2706,12 @@ class Worker:
                     return
                 if _is_storage_admission_error(e):
                     raise
+                delay = _poll_capacity_delay(e, "query_task", self.task_queue)
+                if delay is not None:
+                    self._record_poll_metrics("query", "backpressure", time.perf_counter() - poll_start)
+                    log.debug("query poll capacity backpressure; retrying in %d s", delay)
+                    await asyncio.sleep(delay)
+                    continue
                 self._record_poll_metrics("query", "error", time.perf_counter() - poll_start)
                 log.warning("query poll error: %s", _poll_error_summary(e))
                 await asyncio.sleep(1.0)

{durable_workflow-2.3.0 → durable_workflow-2.3.3/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.3.0
+Version: 2.3.3
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/tests/test_client.py

@@ -2682,6 +2682,34 @@ class TestFailWorkflowTask:
         assert isinstance(request_body["poll_request_id"], str)
         assert request_body["poll_request_id"] != ""
 
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_reaches_worker_after_one_http_request(
+        self, client: Client,
+    ) -> None:
+        requests: list[httpx.Request] = []
+
+        async def refused(method: str, path: str, **kwargs: object) -> httpx.Response:
+            request = httpx.Request(
+                method, "http://localhost:8080" + path,
+                headers=kwargs["headers"], json=kwargs["json"],
+            )
+            requests.append(request)
+            return httpx.Response(429, request=request, json={
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": "workflow_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            })
+
+        with patch.object(client._http, "request", side_effect=refused), pytest.raises(ServerError) as refusal:
+            await client.poll_workflow_task(worker_id="worker-1", task_queue="q1")
+
+        assert len(requests) == 1
+        assert refusal.value.poll_capacity_backpressure_delay("workflow_task", "q1") == 2
+
     @pytest.mark.asyncio
     async def test_poll_workflow_task_response_preserves_no_compatible_status(self, client: Client) -> None:
         response = {

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/tests/test_release_metadata.py

@@ -44,7 +44,7 @@ def test_worker_release_identity_matches_supported_server_and_protocol() -> None
     assert check_release_metadata.VERSION_PATTERN.fullmatch(project["version"]) is not None
     assert release["product-train"] == project["version"]
     assert release["registry-version"] == project["version"].replace("-rc.", "rc")
-    assert release["supported-server-versions"] == "2.4.2"
+    assert check_release_metadata.VERSION_PATTERN.fullmatch(release["supported-server-versions"]) is not None
     assert release["worker-protocol-version"] == PROTOCOL_VERSION == "1.19"
     assert release["durable-selection"] is True
     assert release["durable-selection-minimum-worker-protocol-version"] == "1.19"

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/tests/test_retry_policy.py

@@ -40,6 +40,60 @@ class TestRetryPolicy:
         exc = httpx.HTTPStatusError("rate limited", request=response.request, response=response)
         assert policy.should_retry(exc, attempt=0) is True
 
+    @pytest.mark.parametrize(
+        ("path", "task_kind"),
+        [
+            ("/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/runtime/v1/namespaces/acme/api/worker/workflow-tasks/poll", "workflow_task"),
+            ("/api/worker/activity-tasks/poll", "activity_task"),
+            ("/api/worker/query-tasks/poll", "query_task"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_typed_poll_capacity_refusal_does_not_repeat_transport_request(
+        self, path: str, task_kind: str,
+    ) -> None:
+        policy = RetryPolicy(max_attempts=3, jitter=False)
+        request = httpx.Request(
+            "POST", "http://test" + path,
+            headers={"X-Durable-Workflow-Protocol-Version": "1.19"},
+            json={"task_queue": "q1", "worker_id": "worker-1"},
+        )
+        response = httpx.Response(429, request=request, json={
+            "task": None,
+            "poll_status": "long_poll_capacity_exhausted",
+            "reason": "long_poll_capacity_exhausted",
+            "task_kind": task_kind,
+            "task_queue": "q1",
+            "retryable": True,
+            "retry_after_seconds": 2,
+        })
+        error = httpx.HTTPStatusError("capacity", request=request, response=response)
+        calls = 0
+
+        async def refused() -> None:
+            nonlocal calls
+            calls += 1
+            raise error
+
+        with pytest.raises(httpx.HTTPStatusError):
+            await policy.execute(refused)
+        assert calls == 1
+
+        wrong_kind = httpx.Response(429, request=request, json={
+            **response.json(), "task_kind": task_kind.removesuffix("_task"),
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("wrong kind", request=request, response=wrong_kind), attempt=0,
+        ) is True
+
+        malformed = httpx.Response(429, request=request, json={
+            "reason": "long_poll_capacity_exhausted", "retry_after_seconds": 2,
+        })
+        assert policy.should_retry(
+            httpx.HTTPStatusError("malformed", request=request, response=malformed), attempt=0,
+        ) is True
+
     def test_should_not_retry_4xx_client_error(self) -> None:
         policy = RetryPolicy(max_attempts=3)
         response = httpx.Response(status_code=404, request=httpx.Request("GET", "http://test"))

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/tests/test_storage_admission.py

@@ -39,6 +39,30 @@ def pressure(poll_id: str | None = None, *, reason: str = "storage_pressure", en
     return body
 
 
+def backend_unavailable(request: httpx.Request) -> dict[str, Any]:
+    submitted = json.loads(request.content)
+    path = request.url.path
+    operation = {
+        "/api/worker/workflow-tasks/poll": "poll_workflow_task",
+        "/api/worker/activity-tasks/poll": "poll_activity_task",
+        "/api/worker/query-tasks/poll": "poll_query_task",
+        "/api/worker/register": "register_worker",
+        "/api/worker/heartbeat": "heartbeat_worker",
+    }[path]
+    body: dict[str, Any] = {
+        "reason": "backend_unavailable", "operation": operation, "outcome": "unknown",
+        "worker_id": submitted["worker_id"], "task_queue": submitted.get("task_queue"),
+        "retryable": True, "retry_after_seconds": 1,
+    }
+    if path.endswith("/poll"):
+        body.update({
+            "task": None, "poll_status": "backend_unavailable",
+            "poll_request_id": submitted["poll_request_id"],
+            "retry_same_poll_request_id": True,
+        })
+    return body
+
+
 @contextmanager
 def worker_scope(stop: Callable[[], bool] = lambda: False) -> Iterator[None]:
     token = _worker_storage_admission_stop.set(stop)
@@ -104,6 +128,161 @@ async def test_poll_keeps_identity_after_ambiguous_response_and_pressure(
     assert sum(retry_sleeps) == pytest.approx(5)
 
 
+@pytest.mark.parametrize("kind", ["workflow", "activity", "query", "multiplexed"])
+async def test_backend_outage_preserves_logical_poll_after_transport_budget(
+    kind: str, retry_sleeps: list[float],
+) -> None:
+    requests: list[bytes] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        requests.append(request.content)
+        if len(requests) == 1:
+            raise httpx.ReadTimeout("Response lost after a possible claim.", request=request)
+        if len(requests) <= 5:
+            return httpx.Response(503, json=backend_unavailable(request))
+        return httpx.Response(200, json={"task": {"task_id": "reconciled-claim"}})
+
+    async with client_for(handler) as client:
+        with worker_scope():
+            if kind == "multiplexed":
+                task = await client.poll_workflow_task(
+                    worker_id="backend-worker", task_queue="orders",
+                    task_kinds=("workflow", "update_validation"),
+                )
+            else:
+                task = await getattr(client, f"poll_{kind}_task")(
+                    worker_id="backend-worker", task_queue="orders",
+                )
+    assert task == {"task_id": "reconciled-claim"}
+    assert len(requests) == 6
+    assert len(set(requests)) == 1
+    assert sum(retry_sleeps) == pytest.approx(4)
+
+
+@pytest.mark.parametrize("method,kwargs", [
+    ("register_worker", {
+        "worker_id": "backend-worker", "task_queue": "orders",
+        "capability_manifest": PORTABLE_WORKER_AFFINITY_CAPABILITY_MANIFEST,
+    }),
+    ("heartbeat_worker", {"worker_id": "backend-worker"}),
+])
+async def test_backend_outage_retries_worker_registration_and_heartbeat(
+    method: str, kwargs: dict[str, Any], retry_sleeps: list[float],
+) -> None:
+    requests: list[bytes] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        requests.append(request.content)
+        if len(requests) <= 4:
+            return httpx.Response(503, json=backend_unavailable(request))
+        return httpx.Response(200, json={"recorded": True})
+
+    async with client_for(handler) as client:
+        with worker_scope():
+            result = await getattr(client, method)(**kwargs)
+    assert result == {"recorded": True}
+    assert len(requests) == 5
+    assert len(set(requests)) == 1
+    assert sum(retry_sleeps) == pytest.approx(4)
+
+
+@pytest.mark.parametrize("override", [
+    {"operation": "poll_activity_task"},
+    {"outcome": "failed"}, {"worker_id": "other-worker"},
+    {"task_queue": "other-queue"}, {"retryable": False},
+    {"retry_after_seconds": 0}, {"retry_after_seconds": True},
+    {"retry_after_seconds": "1"}, {"task": {"task_id": "claimed"}},
+    {"poll_status": "empty"}, {"poll_request_id": "another-poll"},
+    {"retry_same_poll_request_id": False},
+])
+async def test_invalid_backend_outage_contract_remains_bounded(
+    override: dict[str, Any], retry_sleeps: list[float],
+) -> None:
+    calls = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json={**backend_unavailable(request), **override})
+
+    async with client_for(handler) as client:
+        with worker_scope(), pytest.raises(ServerError):
+            await client.poll_workflow_task(worker_id="backend-worker", task_queue="orders")
+    assert calls == 1
+    assert not retry_sleeps
+
+
+async def test_legacy_validation_backend_refusal_does_not_repeat_ambiguous_poll(
+    retry_sleeps: list[float],
+) -> None:
+    calls = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json={
+            "reason": "backend_unavailable", "operation": "poll_update_validation_task",
+            "outcome": "unknown", "worker_id": "backend-worker", "task_queue": "orders",
+            "retryable": False, "retry_after_seconds": 1, "task": None,
+            "poll_status": "backend_unavailable", "poll_request_id": None,
+            "retry_same_poll_request_id": True,
+        })
+
+    async with client_for(handler) as client:
+        with worker_scope(), pytest.raises(ServerError):
+            await client.poll_update_validation_task(worker_id="backend-worker", task_queue="orders")
+    assert calls == 1
+    assert not retry_sleeps
+
+
+async def test_direct_backend_outage_poll_remains_bounded(retry_sleeps: list[float]) -> None:
+    calls = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json=backend_unavailable(request))
+
+    async with client_for(handler) as client:
+        with pytest.raises(ServerError):
+            await client.poll_workflow_task(worker_id="backend-worker", task_queue="orders")
+    assert calls == 2
+
+
+async def test_control_backend_error_keeps_finite_transport_retries(retry_sleeps: list[float]) -> None:
+    calls = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json={"reason": "backend_unavailable", "retryable": True})
+
+    async with client_for(handler) as client:
+        with worker_scope(), pytest.raises(ServerError):
+            await client._request("POST", "/workflows", json={})
+    assert calls == 2
+
+
+async def test_shutdown_interrupts_backend_outage_without_new_poll(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls = 0
+    stopped = False
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal calls
+        calls += 1
+        return httpx.Response(503, json=backend_unavailable(request))
+
+    async def stop_on_sleep(delay: float) -> None:
+        nonlocal stopped
+        stopped = True
+
+    monkeypatch.setattr(retry_module, "asyncio", SimpleNamespace(sleep=stop_on_sleep))
+    async with client_for(handler) as client:
+        with worker_scope(lambda: stopped), pytest.raises(ServerError):
+            await client.poll_workflow_task(worker_id="backend-worker", task_queue="orders")
+    assert calls == 1
+
+
 @pytest.mark.parametrize("method,kwargs", [
     ("register_worker", {
         "worker_id": "storage-worker", "task_queue": "orders",

{durable_workflow-2.3.0 → durable_workflow-2.3.3}/tests/test_worker.py

@@ -867,7 +867,60 @@ class TestWorkerRegistration:
         ],
     )
     @pytest.mark.asyncio
-    async def test_poll_warning_omits_large_server_response(
+    async def test_typed_poll_backpressure_waits_without_warning(
+        self,
+        mock_client: AsyncMock,
+        monkeypatch: pytest.MonkeyPatch,
+        caplog: pytest.LogCaptureFixture,
+        poll_method: str,
+        loop_method: str,
+        task_kind: str,
+    ) -> None:
+        worker = Worker(
+            mock_client,
+            task_queue="q1",
+            workflows=[TestWorkflow],
+            activities=[echo_activity],
+        )
+        getattr(mock_client, poll_method).side_effect = ServerError(
+            429,
+            {
+                "task": None,
+                "poll_status": "long_poll_capacity_exhausted",
+                "reason": "long_poll_capacity_exhausted",
+                "task_kind": f"{task_kind}_task",
+                "task_queue": "q1",
+                "retryable": True,
+                "retry_after_seconds": 2,
+            },
+        )
+        outcomes: list[tuple[str, str]] = []
+        monkeypatch.setattr(
+            worker, "_record_poll_metrics",
+            lambda kind, outcome, _duration: outcomes.append((kind, outcome)),
+        )
+
+        async def stop_after_backoff(seconds: float) -> None:
+            assert seconds == 2
+            worker._stop.set()
+
+        monkeypatch.setattr(worker_module.asyncio, "sleep", stop_after_backoff)
+        with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"):
+            await getattr(worker, loop_method)()
+
+        assert outcomes == [(task_kind, "backpressure")]
+        assert not [record for record in caplog.records if record.name == "durable_workflow.worker"]
+
+    @pytest.mark.parametrize(
+        ("poll_method", "loop_method", "task_kind"),
+        [
+            ("poll_workflow_task", "_poll_workflow_tasks", "workflow"),
+            ("poll_activity_task", "_poll_activity_tasks", "activity"),
+            ("poll_query_task", "_poll_query_tasks", "query"),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_malformed_poll_warning_omits_large_server_response(
         self,
         mock_client: AsyncMock,
         monkeypatch: pytest.MonkeyPatch,