cmpnd 0.30.0__tar.gz → 0.31.0__tar.gz

{cmpnd-0.30.0 → cmpnd-0.31.0}/AGENTS.md

@@ -119,6 +119,31 @@ uv run ruff check .                      # lint
 uv run ruff format --check .             # format check
 ```
 
+**Never run pytest processes in parallel** — no `pytest-xdist`, no backgrounding
+one run while starting another. DSPy is memory-heavy on import, and parallel
+pytest crashes the machine. Run suites serially.
+
+**Spec every mock, and assert wire shape over a real client.** A spec-less
+`MagicMock` answers any method with any arguments, so an assertion about how a
+dependency was called is worth exactly the mock's fidelity to it. That shipped
+a live bug: the CLI dispatched a revocation as `httpx.Client.delete(url,
+json=body)`, `httpx` gives `delete` no `json` parameter, the body never left the
+process, and the test asserting the call passed. Two rules, and both are needed:
+
+- Build every double from the real signature — `create_autospec(Thing,
+  instance=True)` or `mock.patch(..., autospec=True)`, as
+  `tests/test_cli/conftest.py` does for `ApiClient`.
+- Where the property under test is what goes on the wire, drive a real
+  `httpx.Client` over an `httpx.MockTransport` that captures the request and
+  assert on the captured bytes — `tests/test_cli/test_api_client.py` is the
+  pattern. Autospec guards only the seam you point it at; the bug above lived
+  one layer below an autospec'd client.
+
+**E2E tests hit a live server.** `tests/e2e/` runs against the real platform
+with real LLMs (`CMPND_E2E_*` keys), never a mock or an in-process fake: its
+job is the integration bugs between the SDK and the real API surface. It is
+local-only by design.
+
 ## Two packages, one published distribution
 
 `cmpnd` (`cmpnd/`) is the SDK. `cmpnd_ingest` (`ingest/cmpnd_ingest/`) is the

{cmpnd-0.30.0 → cmpnd-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cmpnd
-Version: 0.30.0
+Version: 0.31.0
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.30.0 → cmpnd-0.31.0}/cmpnd/_assessments.py

@@ -25,15 +25,17 @@ def create(
     *,
     conclusion: Conclusion = "neutral",
     body: str | None = None,
-    annotations: Sequence[tuple[str, str]] | None = None,
+    annotations: Sequence[tuple[str, str, str]] | None = None,
     api_key: str | None = None,
     endpoint: str | None = None,
 ) -> dict[str, Any]:
     """Record one judgement of a trace and return the created assessment.
 
     ``conclusion`` affirms (``"up"``), disputes (``"down"``), or neither
-    (``"neutral"``, the default). ``annotations`` are ``(span_id, note)`` pairs,
-    each attaching a note to one span of the trace; the server rejects a span id
+    (``"neutral"``, the default). ``annotations`` are ``(kind, ref, note)``
+    tuples where ``kind`` is ``"span"`` (``ref`` is a span id) or ``"output"``
+    (``ref`` is an RFC 6901 JSON pointer such as ``/trends/2``); the tuple order
+    matches the CLI's ``--annotate KIND REF NOTE``. The server rejects a target
     that does not belong to the trace. ``api_key`` / ``endpoint`` override the
     configured credentials for this call.
     """
@@ -41,7 +43,7 @@ def create(
     if body is not None:
         payload["body"] = body
     if annotations:
-        payload["annotations"] = [{"span_id": span, "note": note} for span, note in annotations]
+        payload["annotations"] = [{"kind": kind, "ref": ref, "note": note} for kind, ref, note in annotations]
     return _client(api_key, endpoint).post("/api/v1/assessments", json=payload)
 
 

{cmpnd-0.30.0 → cmpnd-0.31.0}/cmpnd/callback.py

@@ -499,6 +499,11 @@ class _CallbackBase:
         # Track trace roots
         self._root_call_ids: set[str] = set()
 
+        # GEPA's bootstrap tracer temporarily patches the evaluated program's
+        # forward method to return (prediction, bootstrap trace). Track that
+        # wrapper itself so ordinary tuples at roots or under Evaluate stay intact.
+        self._gepa_bootstrap_call_ids: set[str] = set()
+
         # Track spans by trace_id for metric aggregation
         self._trace_spans: dict[str, list[models.SpanBuilder]] = {}
 
@@ -682,6 +687,8 @@ class _ModuleMixin(_CallbackBase):
         # of them, so a module call beneath one roots its own trace. An open
         # trace still wins: a user's `cmpnd.trace` around an Evaluate owns it.
         parent_span = context.get_current_span()
+        if helpers.is_gepa_bootstrap_wrapper(instance):
+            self._gepa_bootstrap_call_ids.add(call_id)
         is_root = parent_span is None or (
             parent_span.span_type == models.SpanType.EVALUATION and context.get_current_trace() is None
         )
@@ -784,11 +791,22 @@ class _ModuleMixin(_CallbackBase):
 
         span = self._call_id_to_span.pop(call_id, None)
         instance = self._call_id_to_instance.pop(call_id, None)
+        is_gepa_capture_boundary = call_id in self._gepa_bootstrap_call_ids
+        self._gepa_bootstrap_call_ids.discard(call_id)
 
         if not span:
             logger.warning(f"No span found for call_id: {call_id}")
             return
 
+        in_optimization = (
+            self._active_optimization_run_id is not None or context.get_current_optimization_run_id() is not None
+        )
+        captured_outputs = (
+            helpers.normalize_gepa_bootstrap_output(outputs, dspy, in_optimization=in_optimization)
+            if is_gepa_capture_boundary
+            else outputs
+        )
+
         # Set end time and status
         span.end_time = datetime.now(timezone.utc)
 
@@ -798,12 +816,12 @@ class _ModuleMixin(_CallbackBase):
             span.status = models.TraceStatus.OK
 
             # Capture outputs
-            if self._capture_outputs and outputs is not None:
-                cleaned_outputs = helpers.clean_outputs(outputs, instance, dspy, self._max_output_field_size)
+            if self._capture_outputs and captured_outputs is not None:
+                cleaned_outputs = helpers.clean_outputs(captured_outputs, instance, dspy, self._max_output_field_size)
                 span.set_outputs(cleaned_outputs)
 
                 # Extract token usage if available
-                self._extract_usage_from_prediction(span, outputs, dspy)
+                self._extract_usage_from_prediction(span, captured_outputs, dspy)
 
         # Pop from context stack
         context.pop_span()
@@ -851,10 +869,10 @@ class _ModuleMixin(_CallbackBase):
             # Skip "FINAL: ..." outputs — these contain the entire RLM result
             # dict and are not meaningful code cell outputs.
             if span.span_type == models.SpanType.RLM:
-                if rlm_parent and rlm_parent.repl_history and outputs is not None:
+                if rlm_parent and rlm_parent.repl_history and captured_outputs is not None:
                     try:
                         last_output = None
-                        traj = outputs.trajectory if hasattr(outputs, "trajectory") else None
+                        traj = captured_outputs.trajectory if hasattr(captured_outputs, "trajectory") else None
                         if traj and hasattr(traj, "entries") and traj.entries:
                             last_output = traj.entries[-1].output
                         elif isinstance(traj, list) and traj:
@@ -893,10 +911,8 @@ class _ModuleMixin(_CallbackBase):
                     trace.error_message = str(exception)
 
                 # Set response preview
-                if outputs is not None and self._capture_outputs:
-                    trace.response_preview = helpers.create_preview(
-                        helpers.clean_outputs(outputs, instance, dspy, self._max_output_field_size)
-                    )
+                if captured_outputs is not None and self._capture_outputs:
+                    trace.response_preview = helpers.create_preview(span.outputs or {})
 
                 # Aggregate span metrics into trace
                 trace_id_str = str(trace.trace_id)
@@ -950,9 +966,9 @@ class _ModuleMixin(_CallbackBase):
                 # store and surface as a phantom output field. Read it back via the
                 # public `cmpnd.trace_id(pred)` accessor. Best-effort: a non-Prediction
                 # or immutable return must never break the call.
-                if outputs is not None:
+                if captured_outputs is not None:
                     try:
-                        outputs._cmpnd_trace_id = trace_id_str
+                        captured_outputs._cmpnd_trace_id = trace_id_str
                     except (AttributeError, TypeError):
                         pass
 

{cmpnd-0.30.0 → cmpnd-0.31.0}/cmpnd/cli/commands/assessments.py

@@ -1,7 +1,10 @@
 """Assessments commands.
 
 An assessment is a person's judgement of a trace: an up/down/neutral
-conclusion, a prose body, and per-span annotations. See the assessments API.
+conclusion, a prose body, and annotations. An annotation attaches a note to a
+typed target, given as ``--annotate KIND REF NOTE``. KIND is ``span`` (REF is a
+span id) or ``output`` (REF is an RFC 6901 JSON pointer such as /trends/2). See
+the assessments API.
 """
 
 from __future__ import annotations
@@ -11,6 +14,9 @@ from typing import Any
 
 from .. import api_client, pagination
 
+# The annotation kinds --annotate accepts; sent through to the wire unchanged.
+_ANNOTATION_KINDS = {"span", "output"}
+
 
 def register(subparsers: _SubParsersAction[ArgumentParser]) -> None:
     parser = subparsers.add_parser("assessments", help="Record and read judgements of traces")
@@ -38,9 +44,10 @@ def register(subparsers: _SubParsersAction[ArgumentParser]) -> None:
         "--annotate",
         dest="annotations",
         action="append",
-        nargs=2,
-        metavar=("SPAN_ID", "NOTE"),
-        help="Attach a note to one span in the trace (repeat for multiple)",
+        nargs=3,
+        metavar=("KIND", "REF", "NOTE"),
+        help="Attach a note to a target: KIND is 'span' (REF is a span id) or "
+        "'output' (REF is an RFC 6901 JSON pointer like /trends/2). Repeat for multiple.",
     )
     create.set_defaults(func=handle_create)
 
@@ -68,8 +75,15 @@ def handle_create(client: api_client.ApiClient, args: Any) -> Any:
     body: dict[str, Any] = {"trace_id": args.trace_id, "conclusion": args.conclusion}
     if args.body:
         body["body"] = args.body
-    if args.annotations:
-        body["annotations"] = [{"span_id": span, "note": note} for span, note in args.annotations]
+    annotations: list[dict[str, str]] = []
+    for kind, ref, note in args.annotations or []:
+        if kind not in _ANNOTATION_KINDS:
+            raise api_client.CliError(
+                f"--annotate KIND must be one of: {', '.join(sorted(_ANNOTATION_KINDS))} (got {kind!r})"
+            )
+        annotations.append({"kind": kind, "ref": ref, "note": note})
+    if annotations:
+        body["annotations"] = annotations
     return client.post("/api/v1/assessments", json=body)
 
 

{cmpnd-0.30.0 → cmpnd-0.31.0}/cmpnd/client.py

@@ -181,9 +181,16 @@ class Client:
         *,
         conclusion: Literal["up", "down", "neutral"] = "neutral",
         body: str | None = None,
-        annotations: list[tuple[str, str]] | None = None,
+        annotations: list[tuple[str, str, str]] | None = None,
     ) -> dict[str, Any]:
-        """Record a judgement of a trace. An omitted conclusion is neutral."""
+        """Record a judgement of a trace. An omitted conclusion is neutral.
+
+        ``annotations`` attach notes to typed targets, each a ``(kind, ref,
+        note)`` tuple — ``kind`` is ``"span"`` (``ref`` is a span id) or
+        ``"output"`` (``ref`` is an RFC 6901 JSON pointer like ``/trends/2``),
+        e.g. ``annotations=[("output", "/trends/2", "the third trend"),
+        ("span", "a1b2c3", "wrong field")]``.
+        """
         self._check_active()
         _exporter_pool.flush(self._config)  # the referenced trace must be durable first
         return _assessments.create(

{cmpnd-0.30.0 → cmpnd-0.31.0}/cmpnd/helpers.py

@@ -45,6 +45,37 @@ def clean_inputs(inputs: dict[str, Any], _dspy: Any, max_field_size: int) -> dic
     return result
 
 
+def is_gepa_bootstrap_wrapper(instance: Any) -> bool:
+    """Return whether DSPy's bootstrap tracer temporarily patched this module."""
+    forward = getattr(instance, "forward", None)
+    function = getattr(forward, "__func__", forward)
+    return (
+        getattr(function, "__module__", None) == "dspy.teleprompt.bootstrap_trace"
+        and getattr(function, "__name__", None) == "patched_forward"
+    )
+
+
+def normalize_gepa_bootstrap_output(outputs: Any, dspy: Any, *, in_optimization: bool) -> Any:
+    """Return the prediction from a GEPA ``(prediction, trace)`` result pair."""
+    if not in_optimization or not isinstance(outputs, tuple) or len(outputs) != 2:
+        return outputs
+
+    prediction, trace = outputs
+    if not isinstance(trace, list):
+        return outputs
+
+    for entry in trace:
+        if (
+            not isinstance(entry, tuple)
+            or len(entry) != 3
+            or not isinstance(entry[0], dspy.Module)
+            or not isinstance(entry[1], dict)
+        ):
+            return outputs
+
+    return prediction
+
+
 def clean_outputs(outputs: Any, _instance: Any, dspy: Any, max_field_size: int) -> dict[str, Any]:
     """Clean and prepare outputs for storage, with per-field truncation."""
     if isinstance(outputs, dspy.Prediction):

{cmpnd-0.30.0 → cmpnd-0.31.0}/docs/cli.md

@@ -123,19 +123,25 @@ warn. On the page-number commands `--offset` must be a whole multiple of
 ## Assessments
 
 `cmpnd assessments` records and reads a person's judgement of a trace: an
-up/down/neutral conclusion, an optional prose body, and per-span annotations.
+up/down/neutral conclusion, an optional prose body, and annotations. An
+annotation attaches a note to a typed target — a span, or a piece of the output
+addressed by a JSON pointer.
 
 > **In development — behind `CMPND_CLI_DEV=1`.** The group is registered only
 > when that env var is set (mirroring `CMPND_CLI_ADMIN`), so it stays hidden from
 > `cmpnd --help` until the flow is finished. The env var only hides the commands;
 > the server still gates each action on the `assessments:*` scope.
 
-- `cmpnd assessments create <trace-id> [--up|--down|--neutral] [-m "prose"] [--annotate <span-id> "note" …]`
+- `cmpnd assessments create <trace-id> [--up|--down|--neutral] [-m "prose"] [--annotate KIND REF NOTE …]`
   — record one judgement. The conclusion flag is optional and mutually exclusive
   (`--up` affirms the trace's output, `--down` disputes it, `--neutral` — the
   default when none is given — does neither). `-m`/`--message` sets the prose
-  body. `--annotate` takes a span id and a note, and repeats to attach a note to
-  more than one span in the trace.
+  body. `--annotate` takes three values: `KIND` is `span` (`REF` is a span id) or
+  `output` (`REF` is an RFC 6901 JSON pointer such as `/trends/2` into the trace's
+  root-span outputs), and `NOTE` is the prose. It repeats, so a judgement can
+  carry several annotations of either kind in the order given. The server rejects
+  a span id absent from the trace and a pointer that does not resolve against the
+  outputs.
 - `cmpnd assessments list (<program> | --trace <trace-id>) [--limit N] [--offset N]`
   — the assessments recorded against a program (the program signature hash is
   positional, in the 16-char hex form `cmpnd signatures list` prints) or against

{cmpnd-0.30.0 → cmpnd-0.31.0}/docs/client.md

@@ -85,6 +85,10 @@ platform's data, so the whole surface is one `dir(sdk)` away:
 - **Assessments** — `sdk.create_assessment(trace_id, conclusion=..., body=..., annotations=...)`,
   `sdk.list_assessments(program=... | trace_id=..., limit=..., offset=...)`,
   `sdk.get_assessment(assessment_id)`, `sdk.delete_assessment(assessment_id)`.
+  `annotations` are `(kind, ref, note)` tuples — `kind` is `"span"` (`ref` is a
+  span id) or `"output"` (`ref` is an RFC 6901 pointer such as `/trends/2`), e.g.
+  `annotations=[("output", "/trends/2", "the third trend"), ("span", "a1b2c3",
+  "wrong field")]`.
 - **Valsets** — `sdk.create_valset(program, name=...)`,
   `sdk.list_valsets(program, limit=..., offset=...)`, `sdk.get_valset(valset_id)`,
   `sdk.list_valset_entries(valset_id, limit=..., offset=...)`,

{cmpnd-0.30.0 → cmpnd-0.31.0}/docs/sdk-instrumentation.md

@@ -86,6 +86,11 @@ The hook methods, in the order you'd expect them to fire for a typical `dspy.Cha
 - **Interpreter (code execution)** — `_InterpreterMixin` (`callback.py:1119`), for the DSPy `CodeInterpreter` hooks DSPy >= 3.3.1 fires around a program running code in the interpreter (`RLM`, `ProgramOfThought`, `CodeAct`, `Flex` — every module reaching it through `_create_interpreter`; `ReAct` / `ReActV2` use no interpreter and fire none). `on_interpreter_execute_start` / `_end` emit one `INTERPRETER_EXECUTE` span per REPL cell — the code in, its captured stdout out, and the `SUBMIT()` result (a DSPy `FinalOutput`) recorded distinctly from stdout. `on_interpreter_tool_call_start` / `_end` synthesize a `TOOL` span for a tool called from interpreter code **only when no real `dspy.Tool` span covered the call** (a user tool routes through `Tool.__call__` and fires `on_tool_*`, so its row is suppressed to avoid a duplicate; the `llm_query` builtins fire no tool hook, so they get the synthesized span). `on_interpreter_startup_*` / `on_interpreter_shutdown_*` emit no span — their duration folds into an `interpreter.startup_ms` / `interpreter.shutdown_ms` attribute on the enclosing span, and an event with no enclosing span (a pooled pre-warm) is dropped.
 - **Evaluate** — `on_evaluate_start` (`callback.py:773`) / `on_evaluate_end` (`callback.py:782`). These are thin forwarders that delegate to `EvalHandler` (see "Eval handling" below).
 
+During a GEPA evaluation, DSPy wraps the program prediction with its internal
+bootstrap trace. The callback recognizes this pair only while an optimization
+is active. It records the prediction on the parent span and leaves the internal
+trace available to GEPA.
+
 Not inheriting the base class costs the free no-op every hook would otherwise
 have. DSPy resolves a handler with a bare `getattr(callback, "on_…")` and does
 not guard for a callback that omits one, so a DSPy release that adds hooks turns

{cmpnd-0.30.0 → cmpnd-0.31.0}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.30.0"
+version = "0.31.0"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.10"

{cmpnd-0.30.0 → cmpnd-0.31.0}/tests/e2e/test_assessments_cli.py

@@ -44,14 +44,18 @@ class TestAssessmentsCliE2E:
 
         client = api_client.ApiClient(CMPND_E2E_API_KEY, E2E_ENDPOINT)
 
-        # create — a down judgement with one span annotation.
+        # create — a down judgement with one span annotation and one output
+        # pointer into the root span's `answer` output.
         created = assessments.handle_create(
             client,
             _ns(
                 trace_id=trace_id,
                 conclusion="down",
                 body="the answer names the office, not the representative",
-                annotations=[[span_id, "this predict returned the wrong field"]],
+                annotations=[
+                    ["span", span_id, "this predict returned the wrong field"],
+                    ["output", "/answer", "the answer field is the wrong value"],
+                ],
             ),
         )
         assert created["id"].startswith("as_")
@@ -59,7 +63,9 @@ class TestAssessmentsCliE2E:
         # The program hash the server copied off the trace round-trips as the same
         # 16-hex string the trace read returned — the CLI passes it through verbatim.
         assert created["program_signature_hash"] == program_hash
-        assert created["annotations"][0]["span_id"] == span_id
+        anns = {(a["kind"], a["ref"]): a for a in created["annotations"]}
+        assert ("span", span_id) in anns
+        assert ("output", "/answer") in anns
 
         # list by program (the 16-hex form) finds the assessment we just created.
         listed = assessments.handle_list(client, _ns(program=program_hash, trace_id=None))

{cmpnd-0.30.0 → cmpnd-0.31.0}/tests/e2e/test_valsets_cli.py

@@ -84,7 +84,12 @@ class TestValsetsCliE2E:
         # An assessment of the trace, then an entry created from it.
         assessment = assessments.handle_create(
             client,
-            _ns(trace_id=trace_id, conclusion="down", body="corrected", annotations=[[span_id, "wrong field"]]),
+            _ns(
+                trace_id=trace_id,
+                conclusion="down",
+                body="corrected",
+                annotations=[["span", span_id, "wrong field"]],
+            ),
         )
         vs = valsets.handle_create(
             client, _ns(program_hash=program_hash, name=f"e2e from-assessment {uuid.uuid4().hex[:8]}")

{cmpnd-0.30.0 → cmpnd-0.31.0}/tests/stress/_readiness.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import time
+import urllib.error
 from collections.abc import Callable, Collection, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Generic, TypeAlias, TypeVar
@@ -99,21 +100,64 @@ def each_scope_has_real_model(
     )
 
 
+def read_tolerating_timeout(
+    read: Callable[[RemainingTimeout], T],
+    unpublished: T,
+) -> Callable[[RemainingTimeout], T]:
+    """Wrap a read so a request timeout counts as "not published yet".
+
+    Only a timeout is tolerated: a read-phase ``TimeoutError`` (a socket timeout
+    after the request was sent) and a connect-phase ``urllib.error.URLError``
+    whose ``.reason`` is a timeout. Inside a bounded poll that is not a failure —
+    the value is simply not visible yet — so return ``unpublished`` and let
+    ``wait_for_publication`` poll again to the deadline, rather than crashing the
+    whole run on one slow request. Other ``URLError`` reasons (DNS, TLS,
+    connection refused) propagate: they are real misconfigurations to surface, not
+    "not published yet", so they must fail rather than be retried to the deadline.
+    Every other exception still surfaces too, so a real HTTP error (which the
+    callers raise as ``AssertionError``) still fails fast. ``socket.timeout`` is an
+    alias of ``TimeoutError`` on Python 3.10+, so the checks cover it. Pair with
+    ``wait_for_publication``'s ``read_timeout`` cap, which bounds how long each
+    such request can block before it is retried.
+    """
+
+    def wrapped(remaining_timeout: RemainingTimeout) -> T:
+        try:
+            return read(remaining_timeout)
+        except TimeoutError:
+            return unpublished
+        except urllib.error.URLError as exc:
+            # URLError also wraps DNS / TLS / connection-refused failures, which
+            # are real problems to surface — not "not published yet". Only a
+            # connect-phase timeout (its reason is a timeout) is tolerated.
+            if isinstance(exc.reason, TimeoutError):
+                return unpublished
+            raise
+
+    return wrapped
+
+
 def wait_for_publication(
     read: Callable[[RemainingTimeout], T],
     ready: Callable[[T], bool],
     *,
     timeout: float = 90.0,
     interval: float = 3.0,
+    read_timeout: float = 10.0,
 ) -> PublicationResult[T]:
     """Read until ``ready`` observes publication within one monotonic deadline.
 
-    ``read`` receives ``remaining_timeout`` and must use a fresh positive value
-    for each blocking HTTP request it makes. The HTTP client uses that value for
-    network-operation timeouts. This helper rejects late results but does not
-    cancel in-flight requests. Exceptions from either callback surface immediately.
-    The returned value remains available for a scoped timeout diagnostic, but a
-    result observed after the deadline is not accepted.
+    ``read`` receives a ``remaining_timeout`` callable and must use a fresh
+    positive value for each blocking HTTP request it makes; the HTTP client uses
+    that value for network-operation timeouts. That value is capped at
+    ``read_timeout`` so one slow request cannot consume the whole window — the loop
+    keeps polling to the deadline instead of blocking for the full budget in a
+    single attempt. ``read`` should treat a request timeout as "not published yet"
+    (wrap it with ``read_tolerating_timeout``) so a slow read is retried rather
+    than fatal; other exceptions from either callback surface immediately. This
+    helper rejects late results but does not cancel in-flight requests. The
+    returned value remains available for a scoped timeout diagnostic, but a result
+    observed after the deadline is not accepted.
     """
     deadline = time.monotonic() + timeout
     last_value: T | None = None
@@ -122,8 +166,14 @@ def wait_for_publication(
         remaining = deadline - time.monotonic()
         return remaining if remaining > 0 else None
 
+    def read_budget() -> float | None:
+        # Bound each request so a single slow read is capped and the loop can poll
+        # again, rather than spending the entire deadline waiting on one request.
+        remaining = remaining_timeout()
+        return None if remaining is None else min(remaining, read_timeout)
+
     while remaining_timeout() is not None:
-        last_value = read(remaining_timeout)
+        last_value = read(read_budget)
         if remaining_timeout() is None:
             return PublicationResult(published=False, value=last_value)
 

{cmpnd-0.30.0 → cmpnd-0.31.0}/tests/stress/test_readiness.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import time
+import urllib.error
 
 import pytest
 
@@ -51,6 +52,87 @@ def test_wait_for_publication_returns_a_late_published_read(monkeypatch):
     assert clock.sleeps == [2.0]
 
 
+def test_wait_for_publication_caps_each_read_below_the_full_budget(monkeypatch):
+    clock = install_fake_clock(monkeypatch)
+    response_timeouts: list[float | None] = []
+    responses = iter([[], [{"trace_id": "trace-1"}]])
+
+    def read(remaining_timeout):
+        response_timeouts.append(remaining_timeout())
+        return next(responses)
+
+    result = _readiness.wait_for_publication(read, bool, timeout=90.0, interval=3.0, read_timeout=10.0)
+
+    assert result.published
+    # Each request is bounded by read_timeout (10.0), never the whole remaining
+    # budget (90.0, then 87.0), so one slow read cannot spend the window.
+    assert response_timeouts == [10.0, 10.0]
+    assert clock.sleeps == [3.0]
+
+
+def test_read_tolerating_timeout_treats_a_read_timeout_as_unpublished():
+    def read(_remaining_timeout):
+        raise TimeoutError("the read operation timed out")
+
+    wrapped = _readiness.read_tolerating_timeout(read, [])
+
+    assert wrapped(lambda: 5.0) == []
+
+
+def test_read_tolerating_timeout_treats_a_connect_phase_timeout_as_unpublished():
+    def read(_remaining_timeout):
+        raise urllib.error.URLError(TimeoutError("timed out"))
+
+    assert _readiness.read_tolerating_timeout(read, [])(lambda: 5.0) == []
+
+
+def test_read_tolerating_timeout_surfaces_a_non_timeout_urlerror():
+    def read(_remaining_timeout):
+        raise urllib.error.URLError(ConnectionRefusedError("refused"))
+
+    with pytest.raises(urllib.error.URLError):
+        _readiness.read_tolerating_timeout(read, [])(lambda: 5.0)
+
+
+def test_read_tolerating_timeout_passes_a_successful_read_through():
+    def read(_remaining_timeout):
+        return ["value"]
+
+    assert _readiness.read_tolerating_timeout(read, [])(lambda: 5.0) == ["value"]
+
+
+def test_read_tolerating_timeout_does_not_swallow_other_errors():
+    def read(_remaining_timeout):
+        raise RuntimeError("POST /api/v1/traces/search -> HTTP 500")
+
+    with pytest.raises(RuntimeError, match="HTTP 500"):
+        _readiness.read_tolerating_timeout(read, [])(lambda: 5.0)
+
+
+def test_wait_for_publication_keeps_polling_after_a_read_timeout(monkeypatch):
+    clock = install_fake_clock(monkeypatch)
+    attempts = {"n": 0}
+
+    def read(_remaining_timeout):
+        attempts["n"] += 1
+        if attempts["n"] == 1:
+            raise TimeoutError("the read operation timed out")
+        return [{"trace_id": "trace-1"}]
+
+    result = _readiness.wait_for_publication(
+        _readiness.read_tolerating_timeout(read, []),
+        bool,
+        timeout=90.0,
+        interval=3.0,
+    )
+
+    # A timed-out first poll is retried to publication, not fatal.
+    assert result.published
+    assert result.value == [{"trace_id": "trace-1"}]
+    assert attempts["n"] == 2
+    assert clock.sleeps == [3.0]
+
+
 def test_wait_for_publication_returns_the_last_missing_value_at_deadline(monkeypatch):
     clock = install_fake_clock(monkeypatch)
     response_timeouts: list[float] = []

{cmpnd-0.30.0 → cmpnd-0.31.0}/tests/stress/test_trace_search.py

@@ -110,9 +110,12 @@ def test_cross_trace_q_search_finds_the_span_token():
         return _search(token, tag, timeout=timeout)
 
     # The projected search reader can finish a request before it folds this
-    # trace. Poll the same scoped read without writing another trace.
+    # trace. Poll the same scoped read without writing another trace. A slow or
+    # unresponsive search read on a loaded staging box is "not published yet",
+    # not a failure: read_tolerating_timeout retries it to the deadline rather
+    # than letting one timed-out request eject the run.
     publication = _readiness.wait_for_publication(
-        search_published_trace,
+        _readiness.read_tolerating_timeout(search_published_trace, []),
         lambda traces: [trace["trace_id"] for trace in traces] == [trace_id],
     )
     found = publication.value or []
@@ -126,5 +129,23 @@ def test_cross_trace_q_search_finds_the_span_token():
 
     # A token no span carries returns nothing under the same tag — proof the match
     # is real token filtering, not the tag scope handing back every tagged trace.
-    absent = _search(f"nomatch{run}", tag)
-    assert absent == [], f"q= matched a token no span carries: {absent}"
+    # Bound it like the positive poll so a slow staging read cannot eject the lane
+    # with an uncaught timeout. Sentinel None (not []) marks a timed-out read,
+    # since [] is the success value; any real response ends the poll, and a
+    # deadline of only-timeouts fails readably rather than falsely confirming
+    # absence.
+    def search_absent_token(remaining_timeout: _readiness.RemainingTimeout) -> list[typing.Any] | None:
+        timeout = remaining_timeout()
+        if timeout is None:
+            return None
+        return _search(f"nomatch{run}", tag, timeout=timeout)
+
+    negative = _readiness.wait_for_publication(
+        _readiness.read_tolerating_timeout(search_absent_token, None),
+        lambda result: result is not None,
+    )
+    assert negative.published, (
+        "negative-token search did not return within 90 seconds (search read timed "
+        "out under load); token absence could not be confirmed"
+    )
+    assert negative.value == [], f"q= matched a token no span carries: {negative.value}"