trodo-python 3.0.1__tar.gz → 3.0.2__tar.gz

{trodo_python-3.0.1 → trodo_python-3.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 3.0.1
+Version: 3.0.2
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-3.0.1 → trodo_python-3.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trodo-python"
-version = "3.0.1"
+version = "3.0.2"
 description = "Trodo Analytics SDK for Python — server-side event tracking"
 readme = "README.md"
 license = { text = "ISC" }

{trodo_python-3.0.1 → trodo_python-3.0.2}/tests/test_long_runs.py

@@ -32,3 +32,30 @@ def test_runs_do_not_share_buffers(http):
     p.enqueue_span(span(1, "a")); p.enqueue_span(span(2, "b")); p.enqueue_span(span(3, "a"))
     assert [rid for rid, _ in http.spans_append] == ["a"]
     assert len(p.get_pending("b")) == 1
+
+
+def big(i, run="r1", kb=300):
+    return TrodoSpan(span_id=f"b{i}", run_id=run, parent_span_id=None, kind="llm", name="chunk",
+                     input=[{"role": "user", "content": "x" * (kb * 1024)}], output="ok", input_tokens=1, output_tokens=1)
+
+
+def test_spans_with_long_prompts_go_in_pieces_under_the_request_limit(http):
+    # 12 spans of ~300 KB: one item would be ~3.6 MB and the sender would drop the whole run.
+    p = TrodoSpanProcessor(http, max_spans_per_run=1000)
+    for i in range(12):
+        p.enqueue_span(big(i))
+    p.ingest_run(TrodoRun(run_id="r1", agent_name="contracts"))
+    import json as _json
+    sizes = [len(_json.dumps(spans)) for _, spans in http.spans_append] + [len(_json.dumps(http.run_ingest[-1]["spans"]))]
+    assert max(sizes) <= 1_000_000 + 320 * 1024, sizes
+    sent = [s["span_id"] for _, spans in http.spans_append for s in spans] + [s["span_id"] for s in http.run_ingest[-1]["spans"]]
+    assert sorted(sent) == sorted(f"b{i}" for i in range(12)) and len(sent) == len(set(sent))
+    assert _aggregate(p.get_pending("r1"))["span_count"] == 0  # all handed over
+
+
+def test_totals_count_spans_sent_ahead_by_size(http):
+    p = TrodoSpanProcessor(http, max_spans_per_run=1000)
+    for i in range(8):
+        p.enqueue_span(big(i, kb=400))
+    assert http.spans_append, "a piece went ahead before the run ended"
+    assert _aggregate(p.get_pending("r1"))["span_count"] == 8

{trodo_python-3.0.1 → trodo_python-3.0.2}/tests/test_sender.py

@@ -271,3 +271,65 @@ def test_agent_decorator_on_an_async_function_records_the_awaited_result():
     item = s.posts[0]["body"]["items"][0]
     assert item["run"]["agent_name"] == "async-bot"
     assert item["run"]["output"] == "HI"     # not "<coroutine object ...>"
+
+
+# ── Pacing (3.0.2): a rate 429 is the server answering, not failing ────────────
+
+def test_rate_paced_items_are_never_given_up_on():
+    # 30 rate refusals over well past max_retry_s, then accepted: nothing lost.
+    paced = (200, None, {"results": [{"status": 429, "deferred": True, "ceiling_units_min": 600, "retry_after_seconds": 1}], "retry_after_ms": 10})
+    s = Server([paced] * 30)
+    sender = Sender(s, **fast(max_retry_s=0.05))
+    sender.enqueue(run("a"))
+    assert wait_for(lambda: sender.stats()["sent"] == 1, timeout=5)
+    assert len(s.posts) == 31
+    assert sender.stats()["dropped"] == 0
+
+
+def test_an_exhausted_allowance_still_gives_up(monkeypatch):
+    import trodo.delivery.sender as m
+    monkeypatch.setattr(m, "_MIN_BACKOFF_S", 0.01)
+    monkeypatch.setattr(m, "_MAX_BACKOFF_S", 0.02)
+    refused = (200, None, {"results": [{"status": 429, "error": "Unit allowance exhausted", "reason": "limit-reached-upgrade-required"}]})
+    s = Server([refused] * 200)
+    sender = Sender(s, **fast(max_retry_s=0.2))
+    sender.enqueue(run("a"))
+    assert wait_for(lambda: sender.stats()["dropped"] == 1, timeout=5)
+    assert sender.stats()["dropped_by_reason"] == {"gave_up": 1}
+
+
+def test_batches_are_sized_to_the_ceiling_a_429_names():
+    # Each run() item is 2 units; a 5-unit ceiling fits two items per batch.
+    s = Server([(429, "0.05", {"error": "Rate ceiling exceeded", "ceiling_units_min": 5, "retry_after_seconds": 1})])
+    sender = Sender(s, **fast(flush_at=50, flush_interval_s=60.0))
+    for rid in "abcdef":
+        sender.enqueue(run(rid))
+    sender.flush(timeout_s=3)
+    assert sender.stats()["sent"] == 6
+    assert [len(p["body"]["items"]) for p in s.posts] == [6, 2, 2, 2]
+
+
+def test_server_held_items_count_as_sent_and_warn_once(capsys):
+    held = (200, None, {"results": [{"status": 202, "queued": True}], "paced": {"queued": 1, "refused": 0, "ceiling_units_min": 600}})
+    s = Server([held, held])
+    sender = Sender(s, **fast())
+    sender.enqueue(run("a"))
+    sender.flush(timeout_s=2)
+    sender.enqueue(run("b"))
+    sender.flush(timeout_s=2)
+    assert sender.stats()["sent"] == 2 and sender.stats()["dropped"] == 0
+    err = capsys.readouterr().err
+    assert err.count("faster than its rate limit (600 units/min)") == 1
+
+
+def test_exit_says_loudly_what_was_not_delivered(capsys):
+    full = (200, None, {"results": [{"status": 429, "deferred": True, "reason": "backlog_full", "ceiling_units_min": 60,
+                                      "retry_after_seconds": 30, "message": "this team's backlog is full"}], "retry_after_ms": 30000})
+    s = Server([full])
+    sender = Sender(s, **fast(exit_flush_s=0.3))
+    sender.enqueue(run("a"))
+    assert wait_for(lambda: len(s.posts) == 1)
+    sender._atexit()
+    err = capsys.readouterr().err
+    assert "1 trace item(s), 2 units, were NOT delivered" in err
+    assert "Trodo said: this team's backlog is full" in err

trodo_python-3.0.2/trodo/_version.py

@@ -0,0 +1 @@
+SDK_VERSION = "3.0.2"

{trodo_python-3.0.1 → trodo_python-3.0.2}/trodo/delivery/sender.py

@@ -14,12 +14,18 @@ and PostHog have converged on (1 Oct 2026 survey):
     the traces being debugged right now survive; drops are counted and
     reported to the server with the next batch.
   - retries while nobody waits: 429 pauses the whole sender until
-    Retry-After (a client over its limit delivers late, not never); 5xx, 408
-    and network errors back off (1 s → 60 s) for up to 10 minutes per item,
-    then it is dropped as ``gave_up``; 413 splits the batch, a single
+    Retry-After (a client over its limit delivers late, not never). A rate
+    429 never counts toward giving up (3.0.2): the server answered, it is
+    only pacing. 5xx, 408, network errors and an exhausted allowance back off
+    (1 s → 60 s) for up to 10 minutes per item, then it is dropped as
+    ``gave_up``. Once a 429 names the team's ceiling (units per minute), a
+    batch carries at most that many units, so each one is admissible; 413 splits the batch, a single
     oversized item is dropped as ``too_large``; any other 4xx is a bad item:
     dropped as ``rejected``, reported once.
-  - exit: ``atexit`` flushes (bounded by ``exit_flush_s``).
+  - exit: ``atexit`` flushes (bounded by ``exit_flush_s``) and says on stderr
+    how many traces were still undelivered, and why, if any were.
+  - the server holds over-rate items itself (202 ``queued``) and stores them
+    at the team's rate; the sender says so on stderr once per 10 minutes.
   - ``immediate`` mode for platforms that freeze between requests (Lambda,
     Cloud Functions, Cloud Run, Vercel): the item is sent now, with short
     retries, on the caller's thread. ``auto`` picks it from the platform's
@@ -124,6 +130,9 @@ class Sender:
         self._stats = {"sent": 0, "dropped": 0, "dropped_by_reason": {}}
         self._unreported = {"count": 0, "by_reason": {}}
         self._warned_drop = False
+        self._unit_cap: Optional[int] = None  # the team's ceiling, once a 429 names it
+        self._last_rate: Optional[Dict[str, Any]] = None
+        self._warned_paced_at = 0.0
         self._thread: Optional[threading.Thread] = None
         if self.mode == "background" and register_atexit:
             atexit.register(self._atexit)
@@ -174,11 +183,13 @@ class Sender:
         with self._lock:
             self._stopped = True
             left = len(self._queue)
+            spans_left = sum(_units_of(q.item) for q in self._queue)
             self._queue.clear()
             self._queue_bytes = 0
             self._wake.notify_all()
         if left:
             self._drop(left, "shutdown", f"{left} items were still queued at shutdown")
+            self._warn_undelivered(left, spans_left)
 
     def stats(self) -> Dict[str, Any]:
         with self._lock:
@@ -229,21 +240,24 @@ class Sender:
     def _take_batch_locked(self) -> List[_Queued]:
         out: List[_Queued] = []
         nbytes = 0
+        units = 0
+        cap = self._unit_cap
         while self._queue and len(out) < self._o["flush_at"]:
             nxt = self._queue[0]
             if out and nbytes + nxt.nbytes > self._o["max_batch_bytes"]:
                 break
+            u = _units_of(nxt.item)
+            if out and cap and units + u > cap:
+                break
             out.append(self._queue.popleft())
             nbytes += nxt.nbytes
+            units += u
         self._queue_bytes -= nbytes
         return out
 
     def _send_batch(self, batch: List[_Queued]) -> None:
-        t = self._now()
         for q in batch:
             q.attempts += 1
-            if q.first_tried_at is None:
-                q.first_tried_at = t
         try:
             status, retry_after, body = self._http.post_raw(
                 endpoints.BATCH, self._envelope(batch), timeout=self._o["request_timeout_s"], gzip_body=True,
@@ -257,8 +271,9 @@ class Sender:
             self._settle(batch, body)
             return
         if status == 429:
+            paced = self._note_rate(body if isinstance(body, dict) else {}, retry_after)
             self._pause(parse_retry_after_s(retry_after, self._now()))
-            self._requeue(batch, 0.0)
+            self._requeue(batch, 0.0, paced=paced)
             return
         if status == 413:
             self._split(batch)
@@ -273,16 +288,23 @@ class Sender:
 
     def _settle(self, batch: List[_Queued], body: Dict[str, Any]) -> None:
         keep: List[_Queued] = []
+        paced: List[_Queued] = []
         rejected = 0
+        queued = 0
         first_error = None
         results = body["results"]
         for i, q in enumerate(batch):
             r = results[i] if i < len(results) and isinstance(results[i], dict) else {"status": 500}
             st = int(r.get("status", 500))
             if 200 <= st < 300:
+                if r.get("queued"):
+                    queued += 1
                 with self._lock:
                     self._stats["sent"] += 1
                 continue
+            if st == 429 and self._note_rate(r, None):
+                paced.append(q)
+                continue
             if st == 429 or st >= 500 or st == 408:
                 keep.append(q)
                 continue
@@ -293,17 +315,60 @@ class Sender:
             first_error = first_error or f"HTTP {st}: {r.get('error', '')}"
         if rejected:
             self._drop(rejected, "rejected", f"{rejected} item(s) rejected by the server ({first_error})")
+        if queued:
+            self._warn_paced(body.get("paced") if isinstance(body.get("paced"), dict) else {})
         ra = body.get("retry_after_ms")
         if ra:
             self._pause(float(ra) / 1000.0)
+        if paced:
+            self._requeue(paced, 0.0, paced=True)
         if keep:
             self._requeue(keep, 0.0 if ra else None)
 
-    def _requeue(self, batch: List[_Queued], wait_s: Optional[float]) -> None:
+    def _note_rate(self, body: Dict[str, Any], retry_after: Optional[str]) -> bool:
+        """Is this 429 the rate ceiling (pacing: retry, never give up) rather
+        than an exhausted allowance? Learns the team's ceiling from it."""
+        ceiling = body.get("ceiling_units_min")
+        rate = bool(retry_after) or body.get("retry_after_seconds") is not None or body.get("deferred") or ceiling is not None
+        if not rate:
+            return False
+        try:
+            c = int(ceiling) if ceiling is not None else 0
+        except (TypeError, ValueError):
+            c = 0
+        with self._lock:
+            if c > 0:
+                self._unit_cap = c
+            self._last_rate = {"ceiling": c or None, "reason": body.get("reason"), "message": body.get("message")}
+        return True
+
+    def _warn_paced(self, paced: Dict[str, Any]) -> None:
+        t = self._now()
+        with self._lock:
+            if t - self._warned_paced_at < 600:
+                return
+            self._warned_paced_at = t
+        c = paced.get("ceiling_units_min")
+        limit = f" ({c} units/min)" if c else ""
+        sys.stderr.write(
+            f"[trodo-python] This team is sending faster than its rate limit{limit}. Trodo is holding the extra traces "
+            "and storing them at that rate, so they will appear late. Contact Trodo for a higher limit.\n"
+        )
+
+    def _requeue(self, batch: List[_Queued], wait_s: Optional[float], paced: bool = False) -> None:
         t = self._now()
         alive: List[_Queued] = []
         for q in batch:
-            if q.first_tried_at is not None and t - q.first_tried_at > self._o["max_retry_s"]:
+            if paced:
+                # The server answered and is only pacing: the give-up clock
+                # (for outages and refusals) starts over.
+                q.first_tried_at = None
+                q.attempts = 0
+                alive.append(q)
+            elif q.first_tried_at is None:
+                q.first_tried_at = t
+                alive.append(q)
+            elif t - q.first_tried_at > self._o["max_retry_s"]:
                 self._drop(1, "gave_up", f"an item was retried for {int(self._o['max_retry_s'])} s without being accepted")
             else:
                 alive.append(q)
@@ -314,7 +379,7 @@ class Sender:
                 self._queue.appendleft(q)
             self._queue_bytes += sum(q.nbytes for q in alive)
             shed = self._shed_locked()
-            attempts = max(q.attempts for q in alive)
+            attempts = max(1, max(q.attempts for q in alive))
             backoff = wait_s if wait_s is not None else min(_MAX_BACKOFF_S, _MIN_BACKOFF_S * 2 ** (attempts - 1)) * (0.5 + random.random() / 2)
             self._not_before = t + backoff
         if shed:
@@ -342,6 +407,7 @@ class Sender:
         one attempt, the status read — a refusal keeps the item like the batch
         path would, instead of the old blocking request that dropped it."""
         keep: List[_Queued] = []
+        paced: List[_Queued] = []
         wait: Optional[float] = None
         for q in batch:
             path, body = _legacy_route(q.item)
@@ -353,14 +419,18 @@ class Sender:
                 with self._lock:
                     self._stats["sent"] += 1
                 continue
+            if status == 429 and self._note_rate(resp if isinstance(resp, dict) else {}, retry_after):
+                paced.append(q)
+                wait = parse_retry_after_s(retry_after, self._now()) or wait
+                continue
             if status is None or status in _TRANSIENT:
                 keep.append(q)
-                if status == 429:
-                    wait = parse_retry_after_s(retry_after, self._now()) or wait
                 continue
             self._drop(1, "too_large" if status == 413 else "rejected", f"POST {path} -> HTTP {status}: {json.dumps(resp)[:200]}")
         if wait is not None:
             self._pause(wait)
+        if paced:
+            self._requeue(paced, 0.0, paced=True)
         if keep:
             self._requeue(keep, 0.0 if wait is not None else None)
 
@@ -472,6 +542,27 @@ class Sender:
             self.flush(self._o["exit_flush_s"])
         except Exception:
             pass
+        with self._lock:
+            left = len(self._queue) + self._in_flight
+            units = sum(_units_of(q.item) for q in self._queue)
+        if left:
+            self._warn_undelivered(left, units)
+
+    def _warn_undelivered(self, items: int, units: int) -> None:
+        """Always on stderr, whatever was said before: data is about to be lost."""
+        with self._lock:
+            rate = self._last_rate if self._paused_until and self._paused_until > self._now() else None
+        why = ""
+        if rate and rate.get("message"):
+            why = f" Trodo said: {rate['message']}"
+        elif rate:
+            c = rate.get("ceiling")
+            why = f" The team is over its rate limit{f' ({c} units/min)' if c else ''} and Trodo could not hold more."
+        sys.stderr.write(
+            f"[trodo-python] {items} trace item(s), {units} units, were NOT delivered before the process exited "
+            f"(waited {self._o['exit_flush_s']:g} s).{why} Call trodo.flush() before exiting (it waits until they are "
+            "delivered), or raise delivery_options exit_flush_s.\n"
+        )
 
     def _log(self, msg: str) -> None:
         if self._debug:
@@ -489,6 +580,14 @@ def _legacy_route(item: Item):
     return endpoints.SPANS_APPEND, {"spans": item["spans"]}
 
 
+def _units_of(item: Any) -> int:
+    """What an item costs against the rate ceiling: one per span, one for the run."""
+    if not isinstance(item, dict):
+        return 0
+    spans = item.get("spans")
+    return (len(spans) if isinstance(spans, list) else 0) + (1 if isinstance(item.get("run"), dict) else 0)
+
+
 def _size_of(item: Any) -> int:
     try:
         return len(json.dumps(item, default=str).encode("utf-8"))

{trodo_python-3.0.1 → trodo_python-3.0.2}/trodo/otel/processor.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import threading
 from dataclasses import dataclass, asdict
+import json
 from typing import Any, Dict, List, Optional, Union
 
 # How long end_run waits for the run's queued spans to send before sending the end.
@@ -97,6 +98,37 @@ class TrodoSpan:
         return {k: v for k, v in asdict(self).items() if v is not None}
 
 
+# A run's spans travel in pieces of at most this many bytes (and at most
+# max_spans_per_run spans). The sender drops any single item over its 4 MB
+# request limit, so before this a run of a few hundred LLM spans with long
+# prompts was dropped whole, run row included.
+_CHUNK_BYTES = 1_000_000
+
+
+def _size(d: Dict[str, Any]) -> int:
+    try:
+        return len(json.dumps(d, default=str))
+    except Exception:
+        return 0
+
+
+def _pieces(dicts: List[Dict[str, Any]], limit: int = _CHUNK_BYTES) -> List[List[Dict[str, Any]]]:
+    """Split span dicts into consecutive pieces of at most ``limit`` bytes (one span always fits a piece)."""
+    out: List[List[Dict[str, Any]]] = []
+    cur: List[Dict[str, Any]] = []
+    size = 0
+    for d in dicts:
+        n = _size(d)
+        if cur and size + n > limit:
+            out.append(cur)
+            cur, size = [], 0
+        cur.append(d)
+        size += n
+    if cur:
+        out.append(cur)
+    return out
+
+
 class _SentSpan:
     """What the run's totals need of a span already sent ahead: no payloads."""
 
@@ -135,6 +167,7 @@ class TrodoSpanProcessor:
         # stores spans before their run arrives) and keeps only what its totals
         # need. Before this, every span after the 1,000th was dropped.
         self._sent_ahead: Dict[str, List[_SentSpan]] = {}
+        self._bucket_bytes: Dict[str, int] = {}
         self._lock = threading.Lock()
         self._shutdown = False
         # run_ids that are "joined" (owned by a remote service) — their spans
@@ -151,6 +184,7 @@ class TrodoSpanProcessor:
             self._joined_runs.discard(run_id)
             self._pending.pop(run_id, None)
             self._sent_ahead.pop(run_id, None)
+            self._bucket_bytes.pop(run_id, None)
 
     def enqueue_span(self, span: TrodoSpan) -> None:
         """Buffer a completed span under its run_id.
@@ -158,19 +192,34 @@ class TrodoSpanProcessor:
         Joined runs flush immediately (append_spans) instead of batching.
         """
         chunk: List[TrodoSpan] = []
+        d = span.to_dict()
+        n = _size(d)
         with self._lock:
             joined = span.run_id in self._joined_runs
             if not joined:
                 bucket = self._pending.setdefault(span.run_id, [])
+                held = self._bucket_bytes.get(span.run_id, 0)
+                # Full by bytes: what is held goes ahead and this span starts the next piece.
+                if bucket and held + n > _CHUNK_BYTES:
+                    chunk = bucket
+                    bucket = []
+                    self._pending[span.run_id] = bucket
+                    held = 0
                 bucket.append(span)
+                held += n
+                # Full by count: everything held goes ahead, this span included.
                 if len(bucket) >= self._max_spans:
-                    chunk = bucket
+                    chunk = chunk + bucket
                     self._pending[span.run_id] = []
+                    held = 0
+                self._bucket_bytes[span.run_id] = held
+                if chunk:
                     self._sent_ahead.setdefault(span.run_id, []).extend(_SentSpan(x) for x in chunk)
         if joined:
-            self._send_spans(span.run_id, [span.to_dict()])
+            self._send_spans(span.run_id, [d])
         elif chunk:
-            self._send_spans(span.run_id, [x.to_dict() for x in chunk])
+            for piece in _pieces([x.to_dict() for x in chunk]):
+                self._send_spans(span.run_id, piece)
 
     def _send_spans(self, run_id: str, spans: List[Dict[str, Any]]) -> None:
         if self._sender is not None:
@@ -191,9 +240,14 @@ class TrodoSpanProcessor:
         with self._lock:
             spans = self._pending.pop(run.run_id, [])
             self._sent_ahead.pop(run.run_id, None)
+            self._bucket_bytes.pop(run.run_id, None)
         payload: Dict[str, Any] = {"run": run.to_dict()}
         if spans:
-            payload["spans"] = [s.to_dict() for s in spans]
+            # All but the last piece go ahead; the run travels with the last.
+            pieces = _pieces([s.to_dict() for s in spans])
+            for piece in pieces[:-1]:
+                self._send_spans(run.run_id, piece)
+            payload["spans"] = pieces[-1]
         if self._sender is not None:
             # Queued: the agent's caller never waits on the network (2.24).
             self._sender.enqueue(payload)
@@ -226,8 +280,9 @@ class TrodoSpanProcessor:
             with self._lock:
                 buffered = self._pending.pop(run_id, None) or []
                 self._sent_ahead.pop(run_id, None)
-            if buffered:
-                self._send_spans(run_id, [s.to_dict() for s in buffered])
+                self._bucket_bytes.pop(run_id, None)
+            for piece in _pieces([s.to_dict() for s in buffered]):
+                self._send_spans(run_id, piece)
             if self._sender is not None:
                 self._sender.flush(END_RUN_FLUSH_S)
         except Exception:
@@ -241,17 +296,18 @@ class TrodoSpanProcessor:
         """Stream spans for a long-running or joined run without finalising."""
         if not spans:
             return
-        self._send_spans(run_id, [s.to_dict() for s in spans])
+        for piece in _pieces([s.to_dict() for s in spans]):
+            self._send_spans(run_id, piece)
 
     def force_flush(self) -> None:
         """Flush any buffered spans that don't have a run finalise yet."""
         with self._lock:
             pending = list(self._pending.items())
             self._pending.clear()
+            self._bucket_bytes.clear()
         for run_id, spans in pending:
-            if not spans:
-                continue
-            self._send_spans(run_id, [s.to_dict() for s in spans])
+            for piece in _pieces([s.to_dict() for s in spans]):
+                self._send_spans(run_id, piece)
 
     def shutdown(self) -> None:
         self._shutdown = True

{trodo_python-3.0.1 → trodo_python-3.0.2}/trodo_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 3.0.1
+Version: 3.0.2
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

trodo_python-3.0.1/trodo/_version.py

@@ -1 +0,0 @@
-SDK_VERSION = "3.0.1"