cmpnd 0.8.2__tar.gz → 0.8.3__tar.gz

{cmpnd-0.8.2 → cmpnd-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmpnd
-Version: 0.8.2
+Version: 0.8.3
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.8.2 → cmpnd-0.8.3}/cmpnd/exporter_http.py

@@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, cast
 
 import httpx
 
-from . import configuration
+from . import configuration, retry
 
 if TYPE_CHECKING:
     from . import models
@@ -55,73 +55,73 @@ def _send_json(client: httpx.Client, method: str, path: str, payload: Any) -> ht
     return client.request(method, path, content=body, headers=_GZIP_HEADERS)
 
 
-# Gateway/overload statuses a data POST should retry rather than count as a hard
-# export error: the backend (or the proxy in front of it) is transiently busy,
-# not rejecting the payload. A retry is safe because serve dedups on ingest —
-# traces by (org_id, trace_id, start_time) and spans by the spans_dedup index —
-# so a re-POST of a request that actually landed can't double-insert.
-_RETRYABLE_STATUS = frozenset({502, 503, 504})
+# Backpressure, not unavailability: the server wants the record shed, so the
+# caller counts it as *dropped* and it is never retried — not even carrying a
+# retryable flag. Named because that exemption is the one thing the contract's
+# body-flag read must not be able to override.
+_SHED_STATUS = 429
 
-# Backpressure, not unavailability: the server's buffer is full and it wants the
-# record dropped, so the caller counts it as *dropped* and we never retry it —
-# not even if the body carried a retryable flag. Named because that exemption is
-# the one thing the body-flag read below must not be able to override.
-_DROP_STATUS = 429
 
+class ExportRetriesExhausted(RuntimeError):
+    """A data POST rode through transient answers until its budget ran out.
 
-def _server_says_retryable(response: httpx.Response) -> bool:
-    """Whether the server explicitly marked this error response retryable.
-
-    The `"retryable": true` error-body field is the server's stable machine
-    signal for "this failed transiently and did not commit — send it again". It
-    is what keeps this side off a hard-coded status list: the server can start
-    signalling a new transient condition without an SDK release. A body that
-    isn't JSON, isn't an object, or lacks the field reads as not-retryable.
+    Distinct from the underlying status error so the exhaustion — not whichever
+    blip happened to be last — is what a caller sees and classifies, and so the
+    count and the window it covered can be named in the message.
     """
-    try:
-        body = response.json()
-    except Exception:
-        return False
-    return isinstance(body, dict) and body.get("retryable") is True
 
 
 def _is_retryable_export_error(exc: BaseException) -> bool:
     """Whether a failed data send is a transient delivery blip worth retrying.
 
-    True for gateway/overload statuses (502/503/504), a response the server
-    explicitly flagged retryable (see `_server_says_retryable`), and httpx
-    transport/timeout errors (connect/read/pool timeouts, connection
-    refused/reset). False for everything else — 4xx (client error, a retry won't
-    help), 429 (backpressure, the caller records it as *dropped* not errored, and
-    no body flag overrides that), and an unflagged 500 (the platform's to answer
-    for, mirroring the is_external_outage boundary). Non-httpx exceptions are not
-    retried (a bug in our own build path shouldn't loop)."""
+    A failed *response* is classified by the platform's retryable-error contract
+    (``cmpnd.retry``): the body's ``retryable: true`` first, then the
+    gateway/overload statuses. So 4xx (client error, a retry won't help) and a
+    plain 500 (the platform's to answer for, mirroring the is_external_outage
+    boundary) fail fast, while a transient condition the server signals on a
+    status this client has no opinion about is honoured.
+
+    **429 is the one status that outranks the flag**, which is why it is checked
+    before the contract rather than left to it. It is backpressure — the server's
+    buffer is full and it wants the record shed, which the caller records as
+    *dropped* rather than errored — so re-sending would amplify the very overload
+    the status exists to relieve. A general classifier cannot make that call: it
+    is a property of this path, where a drop is a legitimate outcome. The
+    contract stays free of the exemption so a poll, which has no drop to fall
+    back on, is not forced into it.
+
+    An httpx transport/timeout error (connect/read/pool timeout, connection
+    refused/reset) is transient too — there is no answer to read, and the send
+    is safe to repeat because serve dedups on ingest (traces by
+    (org_id, trace_id, start_time), spans by the spans_dedup index), so a
+    re-POST of a request that actually landed can't double-insert. Non-httpx
+    exceptions are not retried (a bug in our own build path shouldn't loop)."""
     if isinstance(exc, httpx.HTTPStatusError):
-        status = exc.response.status_code
-        if status == _DROP_STATUS:
+        if exc.response.status_code == _SHED_STATUS:
             return False
-        return status in _RETRYABLE_STATUS or _server_says_retryable(exc.response)
+        return retry.is_retryable(exc.response)
     return isinstance(exc, (httpx.TimeoutException, httpx.TransportError))
 
 
-def _retry_after_seconds(exc: BaseException) -> float | None:
-    """The server's own backoff hint for this failure, in seconds, or None.
+def _may_resend_a_minting_post(exc: BaseException) -> bool:
+    """The same question as above, asked of a POST the server mints a record for.
+
+    The data path may re-send almost anything because serve dedups on ingest, so
+    a repeat of a request that actually landed collapses into the row already
+    there. A start POST has no such safety net — it carries the "new" sentinel
+    and the server allocates the id — so a re-send that the first attempt
+    already completed produces a *second* run, and the caller only ever learns
+    about one of them.
 
-    Only the delta-seconds form of `Retry-After` is honored — that is what the
-    server emits alongside a retryable error. A missing header, the HTTP-date
-    form, or an unparseable value returns None so the caller falls back to its
-    own exponential backoff rather than trusting a number it can't read. A
-    transport error has no response at all and so has no hint.
+    That narrows what earns a retry to the two answers which say the first
+    attempt cannot have landed: the server's own ``retryable``, emitted having
+    already failed the work, and a connection that was never established. A
+    read timeout or a bare 502/504 is excluded precisely because it looks
+    identical whether the server did the work or not.
     """
-    if not isinstance(exc, httpx.HTTPStatusError):
-        return None
-    raw = exc.response.headers.get("retry-after")
-    if not raw:
-        return None
-    try:
-        return max(0.0, float(raw))
-    except ValueError:
-        return None
+    if isinstance(exc, httpx.HTTPStatusError):
+        return retry.signalled_retryable(exc.response)
+    return retry.never_left_the_client(exc)
 
 
 @dataclass
@@ -179,24 +179,6 @@ class BatchExporter:
         # never open a socket. httpx.Client is safe to share across threads.
         self._sync_client: httpx.Client | None = None
         self._sync_client_lock = threading.Lock()
-        # Small courtesy-retry budget for a failed start POST before raising
-        # (fail-loud, no local-id fallback). Instance attrs so tests can tune.
-        self._start_post_attempts = 3
-        self._start_post_backoff = 0.5
-
-        # Bounded retry for the best-effort data POSTs (traces/spans/stats/…):
-        # a transient gateway/overload or transport blip retries with backoff
-        # instead of immediately counting a hard export error. Idempotent on the
-        # serve side (see _is_retryable_export_error). Instance attrs so tests
-        # can zero the backoff. Distinct from the start-post budget above.
-        self._data_post_attempts = 3
-        self._data_post_backoff = 0.5
-        # Ceiling on a single inter-attempt wait, including one the server asked
-        # for via Retry-After. The exporter thread is the only thing draining the
-        # queue, so an over-long wait doesn't just delay this send — it stalls
-        # every queued record behind it and risks the queue filling. Honoring the
-        # server's hint is worth it; honoring an unbounded one is not.
-        self._data_post_max_delay = 5.0
 
         # Guard against registering multiple exit handlers on repeated start()
         self._atexit_registered = False
@@ -209,6 +191,13 @@ class BatchExporter:
         self._exported_count = 0
         self._dropped_count = 0
         self._error_count = 0
+        # What the wire actually answered, so a failed export is diagnosable
+        # past a bare error count: the distribution of non-2xx statuses seen, a
+        # sample of the last error body, and how many transient answers the
+        # retry budget rode through to keep the run alive.
+        self._observed_statuses: dict[int, int] = {}
+        self._last_error_body = ""
+        self._transient_ridden = 0
         self._stats_lock = threading.Lock()
 
     def start(self) -> None:
@@ -377,24 +366,53 @@ class BatchExporter:
     def _post_start_sync(self, path: str, payload: dict[str, Any], *, id_field: str) -> str:
         """POST a start record synchronously and return the server-assigned id.
 
-        Retries a bounded number of times with exponential backoff, then
-        raises — there is no local-id fallback by design.
+        **The server mints the id here** — the payload carries the "new"
+        sentinel — so this POST is not idempotent and what it may re-send is
+        narrower than the data path's. Two answers earn a retry, and only two:
+        the server's own ``retryable``, which is it saying the write did not
+        land, and a connection that was never established, which carried
+        nothing to it. A bare 502/504 does not: that comes from a hop with no
+        idea whether the server processed the request, and re-sending on the
+        guess is how one run start becomes two runs.
+
+        Within that narrowing it rides the full budget, because the window this
+        exists to survive outlasts a couple of courtesy attempts. There is no
+        local-id fallback by design: a run that cannot be started fails loud.
         """
         client = self._sync_http_client()
-        last_exc: Exception | None = None
-        for attempt in range(self._start_post_attempts):
+        tally = retry.TransientTally()
+        while True:
             try:
                 response = _send_json(client, "POST", path, payload)
                 response.raise_for_status()
                 assigned = response.json().get(id_field)
                 if not assigned:
                     raise RuntimeError(f"{path} response missing {id_field}: {response.text}")
+                if tally.count:
+                    self._note_ridden_through(tally.count)
+                    logger.warning(
+                        "cmpnd: rode through %s to start a run via %s", tally.describe(time.monotonic()), path
+                    )
                 return str(assigned)
             except Exception as e:
-                last_exc = e
-                if attempt + 1 < self._start_post_attempts:
-                    time.sleep(self._start_post_backoff * (2**attempt))
-        raise RuntimeError(f"failed to start run via {path} after {self._start_post_attempts} attempts") from last_exc
+                resp = e.response if isinstance(e, httpx.HTTPStatusError) else None
+                if resp is not None:
+                    self._note_status(resp)
+                if not _may_resend_a_minting_post(e):
+                    # RuntimeError whatever the cause, because that is this
+                    # method's documented failure: a run start either yields an
+                    # id or fails loud, and a caller should not have to know
+                    # which layer said no.
+                    raise RuntimeError(f"failed to start run via {path}: {e}") from e
+                tally.record(
+                    str(resp.status_code) if resp is not None else type(e).__name__,
+                    at=time.monotonic(),
+                    sample=str(e),
+                )
+                now = time.monotonic()
+                if tally.exhausted(now) or not self._wait_before_retry(retry.delay_for(tally.count - 1, resp)):
+                    self._note_ridden_through(tally.count)
+                    raise RuntimeError(f"failed to start run via {path}: gave up after {tally.describe(now)}") from e
 
     def export_optimization_progress(self, optimization_run: tracker.OptimizationTracker) -> None:
         """Queue an optimization progress update for export (non-blocking).
@@ -523,45 +541,88 @@ class BatchExporter:
             raise RuntimeError("BackgroundExporter has no HTTP client (not started or already shut down)")
         return self._client
 
+    def _wait_before_retry(self, seconds: float) -> bool:
+        """Wait out a backoff, interruptibly. False means "stop retrying".
+
+        The wait runs on the exporter's own threads, so a plain sleep would hold
+        the process open for the length of the budget at exit. Waiting on the
+        shutdown event instead ends it the moment shutdown is signalled — and
+        means the drain that runs *after* shutdown gets no retries at all, which
+        is the right trade: it must not delay the exit it is part of.
+        """
+        if seconds <= 0:
+            return not self._shutdown.is_set()
+        return not self._shutdown.wait(seconds)
+
     def _send_json_retrying(self, client: httpx.Client, method: str, path: str, payload: Any) -> httpx.Response:
-        """`_send_json` + `raise_for_status`, retrying transient delivery blips.
-
-        Retries `_data_post_attempts` times on a retryable failure — a
-        gateway/overload status, a response the server flagged `retryable`, or a
-        transport/timeout error (see `_is_retryable_export_error`) — then
-        re-raises. The wait between attempts is the server's `Retry-After` hint
-        when it sent one, else exponential backoff, capped either way by
-        `_data_post_max_delay`. A non-retryable failure (4xx, 429, an unflagged
-        500, or a 2xx that somehow raised) propagates on the first attempt, so
-        the caller's existing except blocks still record it (429 → dropped,
-        everything else → error) exactly as before — the only change is that a
-        transient blip now gets a few tries before it counts."""
-        last_exc: Exception | None = None
-        for attempt in range(self._data_post_attempts):
+        """`_send_json` + `raise_for_status`, riding through transient blips.
+
+        Retries a failure the retryable-error contract calls transient (see
+        `_is_retryable_export_error`) on `cmpnd.retry`'s budget: jittered
+        exponential backoff, floored by any `Retry-After` the server sent and
+        capped so no hop can park the exporter, spanning a window sized on the
+        longest transient outage we've measured. A non-retryable failure (4xx,
+        429, a plain 500, or a 2xx that somehow raised) propagates on the first
+        attempt, so the caller's except blocks record it (429 → dropped,
+        everything else → error) with no budget burned.
+
+        An exhausted budget raises ExportRetriesExhausted naming the count and
+        the window, so a run that could not be saved says what it endured.
+        """
+        tally = retry.TransientTally()
+        while True:
             try:
                 response = _send_json(client, method, path, payload)
                 response.raise_for_status()
+                if tally.count:
+                    self._note_ridden_through(tally.count)
+                    logger.warning("cmpnd: rode through %s to %s %s", tally.describe(time.monotonic()), method, path)
                 return response
             except Exception as e:
-                last_exc = e
-                if _is_retryable_export_error(e) and attempt + 1 < self._data_post_attempts:
-                    delay = _retry_after_seconds(e)
-                    if delay is None:
-                        delay = self._data_post_backoff * (2**attempt)
-                    delay = min(delay, self._data_post_max_delay)
-                    logger.debug(
-                        "cmpnd: retrying %s %s after transient error in %.2fs (attempt %d): %s",
-                        method,
-                        path,
-                        delay,
-                        attempt + 1,
-                        e,
-                    )
-                    time.sleep(delay)
-                    continue
-                raise
-        assert last_exc is not None  # unreachable: the loop either returns or raises
-        raise last_exc
+                resp = e.response if isinstance(e, httpx.HTTPStatusError) else None
+                if resp is not None:
+                    self._note_status(resp)
+                if not _is_retryable_export_error(e):
+                    raise
+                tally.record(
+                    str(resp.status_code) if resp is not None else type(e).__name__,
+                    at=time.monotonic(),
+                    sample=str(e),
+                )
+                now = time.monotonic()
+                if tally.exhausted(now) or not self._wait_before_retry(retry.delay_for(tally.count - 1, resp)):
+                    self._note_ridden_through(tally.count)
+                    raise ExportRetriesExhausted(f"{method} {path} gave up after {tally.describe(now)}: {e}") from e
+
+    def _note_status(self, response: httpx.Response) -> None:
+        """Record a non-2xx answer for the end-of-run diagnosis."""
+        with self._stats_lock:
+            self._observed_statuses[response.status_code] = self._observed_statuses.get(response.status_code, 0) + 1
+            body = (response.text or "").strip()
+            if body:
+                self._last_error_body = body[:400]
+
+    def _note_ridden_through(self, count: int) -> None:
+        with self._stats_lock:
+            self._transient_ridden += count
+
+    def failure_summary(self) -> str:
+        """What the wire answered, for a caller reporting a failed export.
+
+        An error count alone can't tell a saturated backend from a rejected
+        payload; the status distribution and one body can.
+        """
+        with self._stats_lock:
+            statuses = dict(self._observed_statuses)
+            ridden = self._transient_ridden
+            body = self._last_error_body
+        if not statuses:
+            return "no non-2xx responses observed"
+        mix = ", ".join(f"{code}={n}" for code, n in sorted(statuses.items()))
+        summary = f"non-2xx observed: {mix}; transient answers ridden through: {ridden}"
+        if body:
+            summary += f"; sample body: {body}"
+        return summary
 
     def _flush_batch(self, batch: list[ExportItem]) -> None:
         """Send a batch of items to the backend with parallel dispatch."""