trodo-python 3.0.0__py3-none-any.whl → 3.0.2__py3-none-any.whl

trodo/_version.py

@@ -1 +1 @@
-SDK_VERSION = "3.0.0"
+SDK_VERSION = "3.0.2"

trodo/delivery/sender.py

@@ -14,12 +14,18 @@ and PostHog have converged on (1 Oct 2026 survey):
     the traces being debugged right now survive; drops are counted and
     reported to the server with the next batch.
   - retries while nobody waits: 429 pauses the whole sender until
-    Retry-After (a client over its limit delivers late, not never); 5xx, 408
-    and network errors back off (1 s → 60 s) for up to 10 minutes per item,
-    then it is dropped as ``gave_up``; 413 splits the batch, a single
+    Retry-After (a client over its limit delivers late, not never). A rate
+    429 never counts toward giving up (3.0.2): the server answered, it is
+    only pacing. 5xx, 408, network errors and an exhausted allowance back off
+    (1 s → 60 s) for up to 10 minutes per item, then it is dropped as
+    ``gave_up``. Once a 429 names the team's ceiling (units per minute), a
+    batch carries at most that many units, so each one is admissible; 413 splits the batch, a single
     oversized item is dropped as ``too_large``; any other 4xx is a bad item:
     dropped as ``rejected``, reported once.
-  - exit: ``atexit`` flushes (bounded by ``exit_flush_s``).
+  - exit: ``atexit`` flushes (bounded by ``exit_flush_s``) and says on stderr
+    how many traces were still undelivered, and why, if any were.
+  - the server holds over-rate items itself (202 ``queued``) and stores them
+    at the team's rate; the sender says so on stderr once per 10 minutes.
   - ``immediate`` mode for platforms that freeze between requests (Lambda,
     Cloud Functions, Cloud Run, Vercel): the item is sent now, with short
     retries, on the caller's thread. ``auto`` picks it from the platform's
@@ -124,6 +130,9 @@ class Sender:
         self._stats = {"sent": 0, "dropped": 0, "dropped_by_reason": {}}
         self._unreported = {"count": 0, "by_reason": {}}
         self._warned_drop = False
+        self._unit_cap: Optional[int] = None  # the team's ceiling, once a 429 names it
+        self._last_rate: Optional[Dict[str, Any]] = None
+        self._warned_paced_at = 0.0
         self._thread: Optional[threading.Thread] = None
         if self.mode == "background" and register_atexit:
             atexit.register(self._atexit)
@@ -174,11 +183,13 @@ class Sender:
         with self._lock:
             self._stopped = True
             left = len(self._queue)
+            spans_left = sum(_units_of(q.item) for q in self._queue)
             self._queue.clear()
             self._queue_bytes = 0
             self._wake.notify_all()
         if left:
             self._drop(left, "shutdown", f"{left} items were still queued at shutdown")
+            self._warn_undelivered(left, spans_left)
 
     def stats(self) -> Dict[str, Any]:
         with self._lock:
@@ -229,21 +240,24 @@ class Sender:
     def _take_batch_locked(self) -> List[_Queued]:
         out: List[_Queued] = []
         nbytes = 0
+        units = 0
+        cap = self._unit_cap
         while self._queue and len(out) < self._o["flush_at"]:
             nxt = self._queue[0]
             if out and nbytes + nxt.nbytes > self._o["max_batch_bytes"]:
                 break
+            u = _units_of(nxt.item)
+            if out and cap and units + u > cap:
+                break
             out.append(self._queue.popleft())
             nbytes += nxt.nbytes
+            units += u
         self._queue_bytes -= nbytes
         return out
 
     def _send_batch(self, batch: List[_Queued]) -> None:
-        t = self._now()
         for q in batch:
             q.attempts += 1
-            if q.first_tried_at is None:
-                q.first_tried_at = t
         try:
             status, retry_after, body = self._http.post_raw(
                 endpoints.BATCH, self._envelope(batch), timeout=self._o["request_timeout_s"], gzip_body=True,
@@ -257,8 +271,9 @@ class Sender:
             self._settle(batch, body)
             return
         if status == 429:
+            paced = self._note_rate(body if isinstance(body, dict) else {}, retry_after)
             self._pause(parse_retry_after_s(retry_after, self._now()))
-            self._requeue(batch, 0.0)
+            self._requeue(batch, 0.0, paced=paced)
             return
         if status == 413:
             self._split(batch)
@@ -273,16 +288,23 @@ class Sender:
 
     def _settle(self, batch: List[_Queued], body: Dict[str, Any]) -> None:
         keep: List[_Queued] = []
+        paced: List[_Queued] = []
         rejected = 0
+        queued = 0
         first_error = None
         results = body["results"]
         for i, q in enumerate(batch):
             r = results[i] if i < len(results) and isinstance(results[i], dict) else {"status": 500}
             st = int(r.get("status", 500))
             if 200 <= st < 300:
+                if r.get("queued"):
+                    queued += 1
                 with self._lock:
                     self._stats["sent"] += 1
                 continue
+            if st == 429 and self._note_rate(r, None):
+                paced.append(q)
+                continue
             if st == 429 or st >= 500 or st == 408:
                 keep.append(q)
                 continue
@@ -293,17 +315,60 @@ class Sender:
             first_error = first_error or f"HTTP {st}: {r.get('error', '')}"
         if rejected:
             self._drop(rejected, "rejected", f"{rejected} item(s) rejected by the server ({first_error})")
+        if queued:
+            self._warn_paced(body.get("paced") if isinstance(body.get("paced"), dict) else {})
         ra = body.get("retry_after_ms")
         if ra:
             self._pause(float(ra) / 1000.0)
+        if paced:
+            self._requeue(paced, 0.0, paced=True)
         if keep:
             self._requeue(keep, 0.0 if ra else None)
 
-    def _requeue(self, batch: List[_Queued], wait_s: Optional[float]) -> None:
+    def _note_rate(self, body: Dict[str, Any], retry_after: Optional[str]) -> bool:
+        """Is this 429 the rate ceiling (pacing: retry, never give up) rather
+        than an exhausted allowance? Learns the team's ceiling from it."""
+        ceiling = body.get("ceiling_units_min")
+        rate = bool(retry_after) or body.get("retry_after_seconds") is not None or body.get("deferred") or ceiling is not None
+        if not rate:
+            return False
+        try:
+            c = int(ceiling) if ceiling is not None else 0
+        except (TypeError, ValueError):
+            c = 0
+        with self._lock:
+            if c > 0:
+                self._unit_cap = c
+            self._last_rate = {"ceiling": c or None, "reason": body.get("reason"), "message": body.get("message")}
+        return True
+
+    def _warn_paced(self, paced: Dict[str, Any]) -> None:
+        t = self._now()
+        with self._lock:
+            if t - self._warned_paced_at < 600:
+                return
+            self._warned_paced_at = t
+        c = paced.get("ceiling_units_min")
+        limit = f" ({c} units/min)" if c else ""
+        sys.stderr.write(
+            f"[trodo-python] This team is sending faster than its rate limit{limit}. Trodo is holding the extra traces "
+            "and storing them at that rate, so they will appear late. Contact Trodo for a higher limit.\n"
+        )
+
+    def _requeue(self, batch: List[_Queued], wait_s: Optional[float], paced: bool = False) -> None:
         t = self._now()
         alive: List[_Queued] = []
         for q in batch:
-            if q.first_tried_at is not None and t - q.first_tried_at > self._o["max_retry_s"]:
+            if paced:
+                # The server answered and is only pacing: the give-up clock
+                # (for outages and refusals) starts over.
+                q.first_tried_at = None
+                q.attempts = 0
+                alive.append(q)
+            elif q.first_tried_at is None:
+                q.first_tried_at = t
+                alive.append(q)
+            elif t - q.first_tried_at > self._o["max_retry_s"]:
                 self._drop(1, "gave_up", f"an item was retried for {int(self._o['max_retry_s'])} s without being accepted")
             else:
                 alive.append(q)
@@ -314,7 +379,7 @@ class Sender:
                 self._queue.appendleft(q)
             self._queue_bytes += sum(q.nbytes for q in alive)
             shed = self._shed_locked()
-            attempts = max(q.attempts for q in alive)
+            attempts = max(1, max(q.attempts for q in alive))
             backoff = wait_s if wait_s is not None else min(_MAX_BACKOFF_S, _MIN_BACKOFF_S * 2 ** (attempts - 1)) * (0.5 + random.random() / 2)
             self._not_before = t + backoff
         if shed:
@@ -342,6 +407,7 @@ class Sender:
         one attempt, the status read — a refusal keeps the item like the batch
         path would, instead of the old blocking request that dropped it."""
         keep: List[_Queued] = []
+        paced: List[_Queued] = []
         wait: Optional[float] = None
         for q in batch:
             path, body = _legacy_route(q.item)
@@ -353,14 +419,18 @@ class Sender:
                 with self._lock:
                     self._stats["sent"] += 1
                 continue
+            if status == 429 and self._note_rate(resp if isinstance(resp, dict) else {}, retry_after):
+                paced.append(q)
+                wait = parse_retry_after_s(retry_after, self._now()) or wait
+                continue
             if status is None or status in _TRANSIENT:
                 keep.append(q)
-                if status == 429:
-                    wait = parse_retry_after_s(retry_after, self._now()) or wait
                 continue
             self._drop(1, "too_large" if status == 413 else "rejected", f"POST {path} -> HTTP {status}: {json.dumps(resp)[:200]}")
         if wait is not None:
             self._pause(wait)
+        if paced:
+            self._requeue(paced, 0.0, paced=True)
         if keep:
             self._requeue(keep, 0.0 if wait is not None else None)
 
@@ -472,6 +542,27 @@ class Sender:
             self.flush(self._o["exit_flush_s"])
         except Exception:
             pass
+        with self._lock:
+            left = len(self._queue) + self._in_flight
+            units = sum(_units_of(q.item) for q in self._queue)
+        if left:
+            self._warn_undelivered(left, units)
+
+    def _warn_undelivered(self, items: int, units: int) -> None:
+        """Always on stderr, whatever was said before: data is about to be lost."""
+        with self._lock:
+            rate = self._last_rate if self._paused_until and self._paused_until > self._now() else None
+        why = ""
+        if rate and rate.get("message"):
+            why = f" Trodo said: {rate['message']}"
+        elif rate:
+            c = rate.get("ceiling")
+            why = f" The team is over its rate limit{f' ({c} units/min)' if c else ''} and Trodo could not hold more."
+        sys.stderr.write(
+            f"[trodo-python] {items} trace item(s), {units} units, were NOT delivered before the process exited "
+            f"(waited {self._o['exit_flush_s']:g} s).{why} Call trodo.flush() before exiting (it waits until they are "
+            "delivered), or raise delivery_options exit_flush_s.\n"
+        )
 
     def _log(self, msg: str) -> None:
         if self._debug:
@@ -489,6 +580,14 @@ def _legacy_route(item: Item):
     return endpoints.SPANS_APPEND, {"spans": item["spans"]}
 
 
+def _units_of(item: Any) -> int:
+    """What an item costs against the rate ceiling: one per span, one for the run."""
+    if not isinstance(item, dict):
+        return 0
+    spans = item.get("spans")
+    return (len(spans) if isinstance(spans, list) else 0) + (1 if isinstance(item.get("run"), dict) else 0)
+
+
 def _size_of(item: Any) -> int:
     try:
         return len(json.dumps(item, default=str).encode("utf-8"))

trodo/otel/processor.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import threading
 from dataclasses import dataclass, asdict
+import json
 from typing import Any, Dict, List, Optional, Union
 
 # How long end_run waits for the run's queued spans to send before sending the end.
@@ -97,6 +98,50 @@ class TrodoSpan:
         return {k: v for k, v in asdict(self).items() if v is not None}
 
 
+# A run's spans travel in pieces of at most this many bytes (and at most
+# max_spans_per_run spans). The sender drops any single item over its 4 MB
+# request limit, so before this a run of a few hundred LLM spans with long
+# prompts was dropped whole, run row included.
+_CHUNK_BYTES = 1_000_000
+
+
+def _size(d: Dict[str, Any]) -> int:
+    try:
+        return len(json.dumps(d, default=str))
+    except Exception:
+        return 0
+
+
+def _pieces(dicts: List[Dict[str, Any]], limit: int = _CHUNK_BYTES) -> List[List[Dict[str, Any]]]:
+    """Split span dicts into consecutive pieces of at most ``limit`` bytes (one span always fits a piece)."""
+    out: List[List[Dict[str, Any]]] = []
+    cur: List[Dict[str, Any]] = []
+    size = 0
+    for d in dicts:
+        n = _size(d)
+        if cur and size + n > limit:
+            out.append(cur)
+            cur, size = [], 0
+        cur.append(d)
+        size += n
+    if cur:
+        out.append(cur)
+    return out
+
+
+class _SentSpan:
+    """What the run's totals need of a span already sent ahead: no payloads."""
+
+    __slots__ = ("input_tokens", "output_tokens", "cost", "kind", "status")
+
+    def __init__(self, span: TrodoSpan) -> None:
+        self.input_tokens = span.input_tokens
+        self.output_tokens = span.output_tokens
+        self.cost = span.cost
+        self.kind = span.kind
+        self.status = span.status
+
+
 class TrodoSpanProcessor:
     """Buffers spans per run_id; flushes to /runs/ingest when the run ends.
 
@@ -118,6 +163,11 @@ class TrodoSpanProcessor:
         self._flush_interval = flush_interval_s
         self._max_spans = max_spans_per_run
         self._pending: Dict[str, List[TrodoSpan]] = {}
+        # A run past max_spans_per_run sends each full chunk ahead (the server
+        # stores spans before their run arrives) and keeps only what its totals
+        # need. Before this, every span after the 1,000th was dropped.
+        self._sent_ahead: Dict[str, List[_SentSpan]] = {}
+        self._bucket_bytes: Dict[str, int] = {}
         self._lock = threading.Lock()
         self._shutdown = False
         # run_ids that are "joined" (owned by a remote service) — their spans
@@ -133,22 +183,43 @@ class TrodoSpanProcessor:
         with self._lock:
             self._joined_runs.discard(run_id)
             self._pending.pop(run_id, None)
+            self._sent_ahead.pop(run_id, None)
+            self._bucket_bytes.pop(run_id, None)
 
     def enqueue_span(self, span: TrodoSpan) -> None:
         """Buffer a completed span under its run_id.
 
         Joined runs flush immediately (append_spans) instead of batching.
         """
+        chunk: List[TrodoSpan] = []
+        d = span.to_dict()
+        n = _size(d)
         with self._lock:
             joined = span.run_id in self._joined_runs
-            if joined:
-                pass
-            else:
+            if not joined:
                 bucket = self._pending.setdefault(span.run_id, [])
-                if len(bucket) < self._max_spans:
-                    bucket.append(span)
+                held = self._bucket_bytes.get(span.run_id, 0)
+                # Full by bytes: what is held goes ahead and this span starts the next piece.
+                if bucket and held + n > _CHUNK_BYTES:
+                    chunk = bucket
+                    bucket = []
+                    self._pending[span.run_id] = bucket
+                    held = 0
+                bucket.append(span)
+                held += n
+                # Full by count: everything held goes ahead, this span included.
+                if len(bucket) >= self._max_spans:
+                    chunk = chunk + bucket
+                    self._pending[span.run_id] = []
+                    held = 0
+                self._bucket_bytes[span.run_id] = held
+                if chunk:
+                    self._sent_ahead.setdefault(span.run_id, []).extend(_SentSpan(x) for x in chunk)
         if joined:
-            self._send_spans(span.run_id, [span.to_dict()])
+            self._send_spans(span.run_id, [d])
+        elif chunk:
+            for piece in _pieces([x.to_dict() for x in chunk]):
+                self._send_spans(span.run_id, piece)
 
     def _send_spans(self, run_id: str, spans: List[Dict[str, Any]]) -> None:
         if self._sender is not None:
@@ -161,15 +232,22 @@ class TrodoSpanProcessor:
 
     def get_pending(self, run_id: str) -> List[TrodoSpan]:
         with self._lock:
-            return list(self._pending.get(run_id, []))
+            # The run's totals cover every span, including those sent ahead.
+            return [*self._sent_ahead.get(run_id, []), *self._pending.get(run_id, [])]
 
     def ingest_run(self, run: TrodoRun) -> None:
         """Flush a run + all its buffered spans in one call."""
         with self._lock:
             spans = self._pending.pop(run.run_id, [])
+            self._sent_ahead.pop(run.run_id, None)
+            self._bucket_bytes.pop(run.run_id, None)
         payload: Dict[str, Any] = {"run": run.to_dict()}
         if spans:
-            payload["spans"] = [s.to_dict() for s in spans]
+            # All but the last piece go ahead; the run travels with the last.
+            pieces = _pieces([s.to_dict() for s in spans])
+            for piece in pieces[:-1]:
+                self._send_spans(run.run_id, piece)
+            payload["spans"] = pieces[-1]
         if self._sender is not None:
             # Queued: the agent's caller never waits on the network (2.24).
             self._sender.enqueue(payload)
@@ -201,8 +279,10 @@ class TrodoSpanProcessor:
         try:
             with self._lock:
                 buffered = self._pending.pop(run_id, None) or []
-            if buffered:
-                self._send_spans(run_id, [s.to_dict() for s in buffered])
+                self._sent_ahead.pop(run_id, None)
+                self._bucket_bytes.pop(run_id, None)
+            for piece in _pieces([s.to_dict() for s in buffered]):
+                self._send_spans(run_id, piece)
             if self._sender is not None:
                 self._sender.flush(END_RUN_FLUSH_S)
         except Exception:
@@ -216,17 +296,18 @@ class TrodoSpanProcessor:
         """Stream spans for a long-running or joined run without finalising."""
         if not spans:
             return
-        self._send_spans(run_id, [s.to_dict() for s in spans])
+        for piece in _pieces([s.to_dict() for s in spans]):
+            self._send_spans(run_id, piece)
 
     def force_flush(self) -> None:
         """Flush any buffered spans that don't have a run finalise yet."""
         with self._lock:
             pending = list(self._pending.items())
             self._pending.clear()
+            self._bucket_bytes.clear()
         for run_id, spans in pending:
-            if not spans:
-                continue
-            self._send_spans(run_id, [s.to_dict() for s in spans])
+            for piece in _pieces([s.to_dict() for s in spans]):
+                self._send_spans(run_id, piece)
 
     def shutdown(self) -> None:
         self._shutdown = True

{trodo_python-3.0.0.dist-info → trodo_python-3.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 3.0.0
+Version: 3.0.2
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-3.0.0.dist-info → trodo_python-3.0.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 trodo/__init__.py,sha256=x7zE63FK8CFJ6ATUvsXxeTkvZUaLKXqAJr99I6nnyZc,27099
-trodo/_version.py,sha256=AsSlr9XWZaS6NVC4bCgB5xuBv1s5UrEYJlMsyJQfpMw,22
+trodo/_version.py,sha256=GmU1Hng0hfRPQUBIrIvzRCIVKBsPLckerj1Erc-Dt1g,22
 trodo/client.py,sha256=eaZtHMtOI-VLgfoqM_KVWMuP_I5Ivs7VFnR0oU96Yqk,23743
 trodo/types.py,sha256=eySgUvCXROG2TxtxgiU0MNr5iH0DEcduK8bmYtTKG44,3138
 trodo/user_context.py,sha256=uHCI2WYoOI3cNwdIUEuZdX4VYSu14pzm348ksn6hn_c,8195
@@ -12,7 +12,7 @@ trodo/api/management.py,sha256=uwhr0ZI1pvguz5-itW1KcgRj0TXM2n4KDa-hGIaBdTA,4023
 trodo/auto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/auto/auto_event_manager.py,sha256=cztuRsRkNoJE5R4NfSfTrTJTGl4jx2Yb-Ncy0aVAPo8,4247
 trodo/delivery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trodo/delivery/sender.py,sha256=urO7m6mRMnSEW7_cNAmJ4A6jtDNxuihZHOxCx1jQScI,21348
+trodo/delivery/sender.py,sha256=kp9kNvS_nJZpRKGD05zXcytwuxCOYD8A27rNJySGqR0,26119
 trodo/managers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/managers/dataset_manager.py,sha256=HTFszK-wKmwjW-aw7xJWr5EKvDpJ_om496MLDr3gelg,13920
 trodo/managers/group_manager.py,sha256=ki3Se3qEoSZfREX63oeDeBmEfZF-ISHLE8azEtLg0tM,3542
@@ -23,7 +23,7 @@ trodo/otel/__init__.py,sha256=yiRFXWUU45bAM2CV37XeO7zf1hmnmjufdP4XO50yEyE,624
 trodo/otel/auto_instrument.py,sha256=M5YVpFoPqn9bQohDFM7yyADrVGJPqFBToJLnBO7nOSA,29029
 trodo/otel/context.py,sha256=Jd0aTc0Q-1dM5kXXinhZD8YtnpdKgvneNiODyboVGKY,1418
 trodo/otel/helpers.py,sha256=XOMWcgZHaq5SQbkFxDaXPE4CDFjn01xjJmJ1vIxvwpw,20730
-trodo/otel/processor.py,sha256=GiMQR3uKHq_sbqXU00MdyarStaBzrZLmPgU1YnlzZb4,8965
+trodo/otel/processor.py,sha256=ezaesBaxgv4-Mx8aHyp3ur41rvHwnQnjCNlPTe8rhjU,12343
 trodo/otel/prompt_trace.py,sha256=-kywRBU0Q2gWntS9HU-P5i7Kc5U-IJU7UVLjkdC2jw4,13593
 trodo/otel/register.py,sha256=7X3AfFQ5YvHGOUmdq8J38LQGvOJFPD907yT3uAp90lI,9607
 trodo/otel/transport.py,sha256=hzZz8gwSMGJ8CxdijmLn1Ljt18owr9XTWy13DLbwYbw,2441
@@ -40,7 +40,7 @@ trodo/session/server_session.py,sha256=McsudEiq33XDq3nqxgzBcUvIjQxCMscwEuAPnYXrT
 trodo/session/session_manager.py,sha256=7ht5LeeZX1HLLfPeNV_a8NkXGbHl3up0CRtCGr3EzjQ,2995
 trodo/util/__init__.py,sha256=Z9c4rPPdKg06Kk3byKheDqksSgR4WNvS475oQ5sNljc,54
 trodo/util/lru.py,sha256=QIsM7s6J_E9ZjiXatSyReZIEeGypTAcgrPWWy8YsRa4,2465
-trodo_python-3.0.0.dist-info/METADATA,sha256=Uq0WsyxJPpE-A8CBpSrafzEvniEf2a__kDIEE9055Ls,25307
-trodo_python-3.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-trodo_python-3.0.0.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
-trodo_python-3.0.0.dist-info/RECORD,,
+trodo_python-3.0.2.dist-info/METADATA,sha256=o047XtyjWl3fOEhJOR12qlWtDqlBelbYvAsbY3wT4xc,25307
+trodo_python-3.0.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+trodo_python-3.0.2.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
+trodo_python-3.0.2.dist-info/RECORD,,