durable-workflow 2.11.0__tar.gz → 2.11.2__tar.gz

{durable_workflow-2.11.0/src/durable_workflow.egg-info → durable_workflow-2.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.11.0
+Version: 2.11.2
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.11.0 → durable_workflow-2.11.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.11.0"
+version = "2.11.2"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -76,8 +76,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.11.0"
-registry-version = "2.11.0"
+product-train = "2.11.2"
+registry-version = "2.11.2"
 supported-server-versions = "2.8.0"
 worker-protocol-version = "1.20"
 control-plane-version = "2"

durable_workflow-2.11.2/src/durable_workflow/_poll_transport.py

@@ -0,0 +1,55 @@
+"""Negotiated long-poll responses and early transport failure diagnostics."""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Mapping
+from typing import Any
+
+import httpx
+
+MEDIA_TYPE = "application/vnd.durable-workflow.poll.v1+json"
+ACCEPT = f"{MEDIA_TYPE}, application/json;q=0.9"
+_SCHEMA = "durable-workflow.poll-transport.v1"
+POLL_PATHS = frozenset(
+    f"/worker/{kind}-tasks/poll" for kind in ("workflow", "activity", "query", "update-validation")
+)
+_HEADERS = frozenset(("retry-after", "x-durable-workflow-protocol-version"))
+log = logging.getLogger(__name__)
+
+
+def unwrap_response(response: httpx.Response) -> httpx.Response:
+    if response.headers.get("content-type", "").split(";", 1)[0].strip().lower() != MEDIA_TYPE:
+        return response
+
+    try:
+        envelope = response.json()
+        if not isinstance(envelope, dict) or envelope.get("schema") != _SCHEMA:
+            raise ValueError("unrecognized poll transport")
+        status, headers, body = envelope.get("status"), envelope.get("headers"), envelope.get("body")
+        if (type(status) is not int or not 200 <= status < 600
+                or response.status_code not in (200, status)
+                or not isinstance(body, dict) or not isinstance(headers, dict)
+                or any(key not in _HEADERS or not isinstance(value, str)
+                       or "\r" in value or "\n" in value for key, value in headers.items())):
+            raise ValueError("invalid poll transport result")
+        # Apply the original status and retry headers before the normal client
+        # error handling. Outer HTTP 200 may have been committed by progress.
+        return httpx.Response(status, headers=headers, json=body, request=response.request)
+    except (ValueError, TypeError) as error:
+        raise httpx.RemoteProtocolError("Invalid worker poll transport response.", request=response.request) from error
+
+
+def warn_early_cutoff(path: str, body: Any, started: float, error: Exception) -> None:
+    wait = body.get("timeout_seconds") if isinstance(body, Mapping) else None
+    observed = time.monotonic() - started
+    if isinstance(wait, bool) or not isinstance(wait, int | float) or wait <= 0 or observed >= wait:
+        return
+    if isinstance(error, httpx.HTTPStatusError) and error.response.status_code not in (502, 504):
+        return
+    log.warning(
+        "Worker long poll %s ended after %.3fs before its configured wait of %.3fs (%s). "
+        "Check proxy and HTTP timeout budgets.",
+        path, observed, wait, type(error).__name__,
+    )

{durable_workflow-2.11.0 → durable_workflow-2.11.2}/src/durable_workflow/client.py

@@ -57,6 +57,10 @@ from ._failure_reporting import (
     prepare_failure_request,
 )
 from ._metric_routes import route_for_metrics as _route_for_metrics
+from ._poll_transport import ACCEPT as _POLL_ACCEPT
+from ._poll_transport import POLL_PATHS as _POLL_PATHS
+from ._poll_transport import unwrap_response as _unwrap_poll_response
+from ._poll_transport import warn_early_cutoff as _warn_poll_cutoff
 from ._query_task_authority import QueryTaskAuthority, current_query_authority
 from .errors import (
     REQUEST_ID_CONFLICT,
@@ -1882,6 +1886,7 @@ class Client:
 
         query_poll = worker and path == "/worker/query-tasks/poll"
         activity_poll = worker and path == "/worker/activity-tasks/poll"
+        progress_poll = worker and method.upper() == "POST" and path in _POLL_PATHS
         if query_poll:
             current_query_authority.set(None)
         if activity_poll:
@@ -1931,15 +1936,34 @@ class Client:
             request_timeout = self.timeout if timeout is None else timeout
             if worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path):
                 request_timeout = min(request_timeout, activity_authority.remaining())
-            resp = await self._http.request(
+            headers = self._discovery_headers() if discovery else self._headers(worker=worker)
+            if progress_poll:
+                headers["Accept"] = _POLL_ACCEPT
+            pending = self._http.request(
                 method,
                 f"/api{path}",
-                headers=self._discovery_headers() if discovery else self._headers(worker=worker),
+                headers=headers,
                 json=request_json,
                 timeout=request_timeout,
             )
-            # Raise HTTPStatusError for 4xx/5xx so retry policy can catch it
-            resp.raise_for_status()
+            try:
+                if progress_poll:
+                    try:
+                        resp = await asyncio.wait_for(pending, timeout=request_timeout)
+                    except asyncio.TimeoutError as error:
+                        raise httpx.ReadTimeout(
+                            "Worker poll exceeded its absolute HTTP attempt deadline.",
+                            request=self._http.build_request(method, f"/api{path}"),
+                        ) from error
+                    resp = _unwrap_poll_response(resp)
+                else:
+                    resp = await pending
+                # Interpret negotiated late errors before the usual retry policy.
+                resp.raise_for_status()
+            except (httpx.TransportError, httpx.HTTPStatusError) as error:
+                if progress_poll:
+                    _warn_poll_cutoff(path, request_json, request_started, error)
+                raise
             return resp
 
         try:

{durable_workflow-2.11.0 → durable_workflow-2.11.2}/src/durable_workflow/worker.py

@@ -259,8 +259,8 @@ class _PollFailureBackoff:
         self._last_summary: str | None = None
         self._last_warning_at = 0.0
 
-    def retry_delay(self, error: Exception, task_kind: str) -> float:
-        delay = random.uniform(self._ceiling / 2, self._ceiling)
+    def retry_delay(self, error: Exception, task_kind: str, *, minimum_delay: float = 0.0) -> float:
+        delay = max(minimum_delay, random.uniform(self._ceiling / 2, self._ceiling))
         self._ceiling = min(30.0, self._ceiling * 2)
         summary = _poll_error_summary(error)
         now = time.monotonic()
@@ -280,6 +280,28 @@ def _poll_capacity_delay(error: BaseException, task_kind: str, task_queue: str)
     return error.poll_capacity_backpressure_delay(task_kind, task_queue)
 
 
+def _backend_lock_poll_delay(
+    error: Exception, task_kind: str, task_queue: str, namespace: object,
+) -> int | None:
+    if not isinstance(error, ServerError) or error.status != 503:
+        return None
+    body = error.body
+    if (
+        not isinstance(body, dict) or not isinstance(namespace, str) or not namespace
+        or body.get("reason") != "backend_lock_pressure"
+        or body.get("poll_status") != "backend_lock_pressure"
+        or "task" not in body or body["task"] is not None
+        or body.get("task_kind") != task_kind + "_task"
+        or body.get("task_queue") != task_queue or body.get("namespace") != namespace
+        or ("retryable" in body and body["retryable"] is not True)
+        or ("request_admitted" in body and body["request_admitted"] is not False)
+        or type(body.get("retry_after_seconds")) is not int
+        or not 0 < body["retry_after_seconds"] <= 60
+    ):
+        return None
+    return int(body["retry_after_seconds"])
+
+
 def _command_payload_codec(codec: object) -> str:
     if codec is None:
         return serializer.AVRO_CODEC
@@ -4468,6 +4490,7 @@ class Worker:
     ) -> WorkflowExecution:
         deadline = time.monotonic() + timeout
         next_task_kind = "workflow"
+        failures = {"workflow": _PollFailureBackoff(), "activity": _PollFailureBackoff()}
 
         while not self._stop.is_set():
             desc = await self.client.describe_workflow(workflow_id)
@@ -4494,9 +4517,14 @@ class Worker:
                         task_kinds=self._workflow_poll_task_kinds(),
                         history_page_size=WORKFLOW_HISTORY_PAGE_SIZE,
                     )
-                except BaseException:
+                except BaseException as error:
                     self._release_workflow_capacity()
+                    if isinstance(error, Exception) and await self._retry_run_until_poll_refusal(
+                        error, "workflow", poll_start, deadline, failures["workflow"],
+                    ):
+                        continue
                     raise
+                failures["workflow"].reset()
                 if self._stop.is_set():
                     self._release_workflow_capacity()
                     raise asyncio.CancelledError
@@ -4543,12 +4571,20 @@ class Worker:
                 await asyncio.sleep(poll_interval)
                 continue
             poll_start = time.perf_counter()
-            task = await self.client.poll_activity_task(
-                worker_id=self.worker_id,
-                task_queue=self.task_queue,
-                timeout=self._poll_timeout,
-                build_id=self.build_id,
-            )
+            try:
+                task = await self.client.poll_activity_task(
+                    worker_id=self.worker_id,
+                    task_queue=self.task_queue,
+                    timeout=self._poll_timeout,
+                    build_id=self.build_id,
+                )
+            except Exception as error:
+                if await self._retry_run_until_poll_refusal(
+                    error, "activity", poll_start, deadline, failures["activity"],
+                ):
+                    continue
+                raise
+            failures["activity"].reset()
             if self._stop.is_set():
                 raise asyncio.CancelledError
             self._record_poll_metrics(
@@ -4566,6 +4602,28 @@ class Worker:
 
         raise asyncio.CancelledError
 
+    async def _retry_run_until_poll_refusal(
+        self, error: Exception, kind: str, poll_start: float, deadline: float,
+        failures: _PollFailureBackoff,
+    ) -> bool:
+        delay = _backend_lock_poll_delay(
+            error, kind, self.task_queue, getattr(self.client, "namespace", None),
+        )
+        if delay is None:
+            return False
+        self._record_poll_metrics(kind, "error", time.perf_counter() - poll_start)
+        backoff = failures.retry_delay(error, kind, minimum_delay=delay)
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise asyncio.TimeoutError
+        # Both the public caller timer and this local wait retain their original
+        # deadline. Stop interrupts the backoff without another poll or dispatch.
+        with contextlib.suppress(asyncio.TimeoutError):
+            await asyncio.wait_for(self._stop.wait(), timeout=min(backoff, remaining))
+        if time.monotonic() >= deadline:
+            raise asyncio.TimeoutError
+        return True
+
     async def _run_until_activity_task(self, task: dict[str, Any]) -> None:
         task_start = time.perf_counter()
         outcome = "error"

{durable_workflow-2.11.0 → durable_workflow-2.11.2/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.11.0
+Version: 2.11.2
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.11.0 → durable_workflow-2.11.2}/src/durable_workflow.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ src/durable_workflow/_cancellation_scope_history.py
 src/durable_workflow/_cooperative_cancellation.py
 src/durable_workflow/_failure_reporting.py
 src/durable_workflow/_metric_routes.py
+src/durable_workflow/_poll_transport.py
 src/durable_workflow/_prepared_local_activity.py
 src/durable_workflow/_query_task_authority.py
 src/durable_workflow/_sticky_workflow_cache.py
@@ -104,6 +105,7 @@ tests/test_parallel_saga_authoring.py
 tests/test_patch_history.py
 tests/test_pending_operation_replay.py
 tests/test_poll_failure_backoff.py
+tests/test_poll_transport.py
 tests/test_prepared_local_activity_client.py
 tests/test_prepared_local_activity_groups.py
 tests/test_prepared_local_activity_policies.py

durable_workflow-2.11.2/tests/test_poll_transport.py

@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager, suppress
+from typing import Any
+
+import httpx
+import pytest
+
+from durable_workflow._poll_transport import MEDIA_TYPE, unwrap_response
+from durable_workflow.client import Client
+from durable_workflow.errors import InvalidArgument, ServerError
+from durable_workflow.retry_policy import TransportRetryPolicy
+
+
+def envelope(status: int, body: Any, **changes: Any) -> dict[str, Any]:
+    return {"schema": "durable-workflow.poll-transport.v1", "status": status, "headers": {},
+            "body": body, **changes}
+
+
+def wire_response(body: Any, *, status: int = 200) -> httpx.Response:
+    return httpx.Response(status, headers={"content-type": MEDIA_TYPE.upper() + "; charset=utf-8"},
+                          json=body, request=httpx.Request("POST", "http://fixture/api/worker/workflow-tasks/poll"))
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["workflow", "activity", "query", "update-validation"])
+@pytest.mark.parametrize("negotiated", [False, True])
+async def test_poll_negotiation_accepts_legacy_and_progress(kind: str, negotiated: bool) -> None:
+    async def serve(request: httpx.Request) -> httpx.Response:
+        assert request.headers["accept"] == MEDIA_TYPE + ", application/json;q=0.9"
+        data = {"task": None, "poll_status": "empty"}
+        if negotiated:
+            return httpx.Response(200, headers={"content-type": MEDIA_TYPE},
+                                  content=b"\n\n" + json.dumps(envelope(200, data)).encode())
+        return httpx.Response(200, json=data)
+
+    async with Client("http://fixture") as client:
+        await client._http.aclose()
+        client._http = httpx.AsyncClient(base_url="http://fixture", transport=httpx.MockTransport(serve))
+        poll = getattr(client, f"poll_{kind.replace('-', '_')}_task")
+        assert await poll(worker_id="fixture-worker", task_queue="fixture-queue", timeout=0) is None
+
+
+@pytest.mark.asyncio
+async def test_non_poll_request_keeps_ordinary_response() -> None:
+    async def serve(request: httpx.Request) -> httpx.Response:
+        assert MEDIA_TYPE not in request.headers.get("accept", "")
+        return httpx.Response(200, json={"healthy": True})
+
+    async with Client("http://fixture") as client:
+        await client._http.aclose()
+        client._http = httpx.AsyncClient(base_url="http://fixture", transport=httpx.MockTransport(serve))
+        assert await client.health() == {"healthy": True}
+
+
+def test_unwrap_preserves_task_values_and_original_headers() -> None:
+    body = {"task": {"counter": 9007199254740993, "value": 1.0, "payload": {"codec": "avro", "data": "AA=="}}}
+    response = unwrap_response(wire_response(envelope(200, body, headers={
+        "x-durable-workflow-protocol-version": "1.20", "retry-after": "2",
+    })))
+    assert response.json() == body
+    assert response.headers["x-durable-workflow-protocol-version"] == "1.20"
+    assert response.headers["retry-after"] == "2"
+    assert len(response.content) == int(response.headers["content-length"])
+
+
+@pytest.mark.parametrize("change", [
+    {"schema": "unknown"}, {"status": True}, {"status": 199}, {"status": 600},
+    {"body": []}, {"headers": []}, {"headers": {"retry-after": 2}},
+    {"headers": {"authorization": "not-allowed"}}, {"headers": {"retry-after": "2\r\nX-Test: bad"}},
+])
+def test_invalid_transport_is_not_silently_treated_as_an_empty_poll(change: dict[str, Any]) -> None:
+    response = wire_response(envelope(200, {"task": None}) | change)
+    with pytest.raises(httpx.RemoteProtocolError, match="Invalid worker poll transport"):
+        unwrap_response(response)
+
+
+def test_outer_http_error_cannot_be_rewritten_as_success() -> None:
+    response = wire_response(envelope(200, {"task": None}), status=503)
+    with pytest.raises(httpx.RemoteProtocolError):
+        unwrap_response(response)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [409, 422, 429, 503])
+async def test_late_errors_retain_typed_error_and_retry_semantics(status: int) -> None:
+    attempts = []
+    retry_policy = TransportRetryPolicy(max_attempts=2, initial_backoff_seconds=0, jitter=False)
+
+    async def serve(request: httpx.Request) -> httpx.Response:
+        attempts.append(json.loads(request.content)["poll_request_id"])
+        reason = "long_poll_capacity_exhausted" if status == 429 else "fixture_late_failure"
+        body = {"error": "fixture failure", "reason": reason, "retryable": True}
+        if status == 429:
+            body.update({"task": None, "poll_status": reason, "task_kind": "workflow_task",
+                         "task_queue": "fixture-queue", "retry_after_seconds": 2})
+        return httpx.Response(200, headers={"content-type": MEDIA_TYPE}, json=envelope(
+            status, body,
+            headers={"retry-after": "2"},
+        ))
+
+    async with Client("http://fixture", retry_policy=retry_policy) as client:
+        await client._http.aclose()
+        client._http = httpx.AsyncClient(base_url="http://fixture", transport=httpx.MockTransport(serve))
+        with pytest.raises(InvalidArgument if status == 422 else ServerError) as raised:
+            await client.poll_workflow_task(worker_id="fixture-worker", task_queue="fixture-queue", timeout=0)
+    if isinstance(raised.value, ServerError):
+        assert raised.value.status == status
+        assert raised.value.reason() == ("long_poll_capacity_exhausted" if status == 429 else "fixture_late_failure")
+    assert len(attempts) == (2 if status == 503 else 1)
+    assert len(set(attempts)) == 1
+
+
+@asynccontextmanager
+async def progressing_peer(
+    *, complete_after: float, requests: list[dict[str, Any]] | None = None,
+) -> AsyncIterator[tuple[str, list[asyncio.Event]]]:
+    closed = asyncio.Event()
+    started = asyncio.Event()
+    handlers: set[asyncio.Task[None]] = set()
+
+    async def serve(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        task = asyncio.current_task()
+        assert task is not None
+        handlers.add(task)
+        try:
+            headers = await reader.readuntil(b"\r\n\r\n")
+            for header in headers.split(b"\r\n"):
+                if header.lower().startswith(b"content-length:"):
+                    body = await reader.readexactly(int(header.split(b":", 1)[1]))
+                    if requests is not None:
+                        requests.append(json.loads(body))
+            writer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n"
+                         + f"Content-Type: {MEDIA_TYPE}\r\n\r\n".encode())
+            started.set()
+            finish = time.monotonic() + complete_after
+            while time.monotonic() < finish:
+                writer.write(b"1\r\n\n\r\n")
+                await writer.drain()
+                if reader.at_eof():
+                    return
+                await asyncio.sleep(0.01)
+            if not reader.at_eof():
+                data = json.dumps(envelope(200, {"task": None, "poll_status": "empty"})).encode()
+                writer.write(f"{len(data):x}\r\n".encode() + data + b"\r\n0\r\n\r\n")
+                await writer.drain()
+        except (ConnectionError, asyncio.IncompleteReadError):
+            pass
+        finally:
+            writer.close()
+            with suppress(ConnectionError):
+                await writer.wait_closed()
+            handlers.discard(task)
+            closed.set()
+
+    server = await asyncio.start_server(serve, "127.0.0.1", 0)
+    try:
+        assert server.sockets
+        yield f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}", [started, closed]
+    finally:
+        server.close()
+        await server.wait_closed()
+        for task in list(handlers):
+            task.cancel()
+        if handlers:
+            await asyncio.gather(*handlers, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["workflow", "activity", "query", "update-validation"])
+async def test_progress_cannot_renew_the_absolute_attempt_deadline(
+    kind: str, caplog: pytest.LogCaptureFixture,
+) -> None:
+    async with (
+        progressing_peer(complete_after=0.6) as (endpoint, events),
+        Client(endpoint, retry_policy=TransportRetryPolicy(max_attempts=1)) as client,
+    ):
+        started = time.monotonic()
+        request = client._request("POST", f"/worker/{kind}-tasks/poll", worker=True,
+                                  json={"timeout_seconds": 30}, timeout=0.12)
+        with caplog.at_level(logging.WARNING), pytest.raises(httpx.ReadTimeout, match="absolute HTTP attempt"):
+            await asyncio.wait_for(request, timeout=2)
+        assert time.monotonic() - started < 0.5
+        await asyncio.wait_for(events[1].wait(), timeout=1)
+        assert "configured wait of 30.000s" in caplog.text
+        assert "ended after" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_public_poll_keeps_its_retry_budget_and_logical_identity() -> None:
+    requests: list[dict[str, Any]] = []
+    async with (
+        progressing_peer(complete_after=1.2, requests=requests) as (endpoint, _events),
+        Client(endpoint, retry_policy=TransportRetryPolicy(max_attempts=1)) as client,
+    ):
+        started = time.monotonic()
+        result = await client.poll_workflow_task_response(
+            worker_id="fixture-worker", task_queue="fixture-queue", timeout=0,
+        )
+        assert result == {"task": None, "poll_status": "timeout"}
+        assert 1.8 < time.monotonic() - started < 3
+        assert len(requests) == 2
+        assert len({request["poll_request_id"] for request in requests}) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [502, 504])
+async def test_gateway_cutoff_warns_with_observed_and_configured_wait(
+    status: int, caplog: pytest.LogCaptureFixture,
+) -> None:
+    async def serve(_request: httpx.Request) -> httpx.Response:
+        return httpx.Response(status, text="fixture gateway error")
+
+    async with Client("http://fixture", retry_policy=TransportRetryPolicy(max_attempts=1)) as client:
+        await client._http.aclose()
+        client._http = httpx.AsyncClient(base_url="http://fixture", transport=httpx.MockTransport(serve))
+        with caplog.at_level(logging.WARNING), pytest.raises(ServerError) as raised:
+            await client.poll_workflow_task(worker_id="fixture-worker", task_queue="fixture-queue", timeout=30)
+    assert raised.value.status == status
+    assert "configured wait of 30.000s" in caplog.text
+    assert "ended after" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_progress_completes_inside_the_attempt_deadline() -> None:
+    async with (
+        progressing_peer(complete_after=0.12) as (endpoint, _events),
+        Client(endpoint, retry_policy=TransportRetryPolicy(max_attempts=1)) as client,
+    ):
+        result = await client._request("POST", "/worker/workflow-tasks/poll", worker=True,
+                                     json={"timeout_seconds": 1}, timeout=0.5)
+        assert result == {"task": None, "poll_status": "empty"}
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_a_progressing_request_without_a_cutoff_warning(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    async with (
+        progressing_peer(complete_after=1) as (endpoint, events),
+        Client(endpoint, retry_policy=TransportRetryPolicy(max_attempts=1)) as client,
+    ):
+        request = asyncio.create_task(client._request("POST", "/worker/workflow-tasks/poll", worker=True,
+                                                     json={"timeout_seconds": 30}, timeout=5))
+        await asyncio.wait_for(events[0].wait(), timeout=1)
+        request.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await request
+        await asyncio.wait_for(events[1].wait(), timeout=1)
+        assert "ended after" not in caplog.text