tokenaudit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. tokenaudit/__init__.py +3 -0
  2. tokenaudit/__main__.py +152 -0
  3. tokenaudit/ci/__init__.py +1 -0
  4. tokenaudit/counterfactual/__init__.py +1 -0
  5. tokenaudit/detect/__init__.py +15 -0
  6. tokenaudit/detect/base.py +96 -0
  7. tokenaudit/detect/corpus_record.json +56 -0
  8. tokenaudit/detect/engine.py +87 -0
  9. tokenaudit/detect/findings.py +86 -0
  10. tokenaudit/detect/rules/__init__.py +1 -0
  11. tokenaudit/detect/rules/breakpoint_limit.py +102 -0
  12. tokenaudit/detect/rules/context_bloat.py +105 -0
  13. tokenaudit/detect/rules/lookback_overflow.py +120 -0
  14. tokenaudit/detect/rules/model_switch.py +107 -0
  15. tokenaudit/detect/rules/prefix_drift.py +157 -0
  16. tokenaudit/detect/rules/retry_storm.py +102 -0
  17. tokenaudit/detect/rules/sub_minimum.py +112 -0
  18. tokenaudit/detect/rules/tool_schema_churn.py +120 -0
  19. tokenaudit/detect/rules/ttl_expiry.py +121 -0
  20. tokenaudit/examples/sample.trace.json +512 -0
  21. tokenaudit/ingest/__init__.py +39 -0
  22. tokenaudit/ingest/containers.py +172 -0
  23. tokenaudit/ingest/dedup.py +64 -0
  24. tokenaudit/ingest/dialects.py +258 -0
  25. tokenaudit/ingest/quality.py +65 -0
  26. tokenaudit/ingest/sessionize.py +47 -0
  27. tokenaudit/model/__init__.py +5 -0
  28. tokenaudit/model/records.py +99 -0
  29. tokenaudit/py.typed +0 -0
  30. tokenaudit/report/__init__.py +14 -0
  31. tokenaudit/report/corpus_record.py +35 -0
  32. tokenaudit/report/health.py +86 -0
  33. tokenaudit/report/html.py +152 -0
  34. tokenaudit/rulebook/__init__.py +238 -0
  35. tokenaudit/rulebook/anthropic.json +36 -0
  36. tokenaudit/rulebook/google.json +23 -0
  37. tokenaudit/rulebook/openai.json +45 -0
  38. tokenaudit-0.1.0.dist-info/METADATA +111 -0
  39. tokenaudit-0.1.0.dist-info/RECORD +42 -0
  40. tokenaudit-0.1.0.dist-info/WHEEL +4 -0
  41. tokenaudit-0.1.0.dist-info/entry_points.txt +2 -0
  42. tokenaudit-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,105 @@
1
+ """Context bloat: the whole conversation re-sent every call, uncached.
2
+
3
+ The only usage-driven detector in the library: it needs no message content,
4
+ so it still works on traces captured with content capture off (the OTel
5
+ default). Fires when input grows monotonically across an unmanaged session
6
+ — no cache markers anywhere — while cache reads stay near zero, i.e. every
7
+ repeated token is billed at full price, forever, by construction.
8
+
9
+ Sessions that use cache markers are excluded: growth under markers has
10
+ specific failure modes (lookback overflow, breakpoint budget) with their
11
+ own detectors, and blaming "bloat" there would suggest the wrong fix.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from itertools import pairwise
17
+
18
+ from tokenaudit.detect.base import Abstain, Detector
19
+ from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
20
+ from tokenaudit.model.records import Session
21
+ from tokenaudit.rulebook import Rulebook
22
+
23
+ _MIN_REQUESTS = 5
24
+ _MONOTONIC_FRACTION = 0.8
25
+ _GROWTH_FACTOR = 2.0
26
+ _MAX_READ_RATIO = 0.3
27
+
28
+
29
+ class ContextBloatDetector(Detector):
30
+ detector_id = "context_bloat"
31
+
32
+ def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
33
+ requests = [r for r in session.requests if r.usage.input_total is not None]
34
+ if len(requests) < _MIN_REQUESTS:
35
+ raise Abstain(f"needs at least {_MIN_REQUESTS} requests with input token counts")
36
+ if any(block.cache_marker for r in requests if r.blocks for block in r.blocks):
37
+ return [] # marker-managed growth has its own, more specific detectors
38
+
39
+ inputs = [r.usage.input_total or 0 for r in requests]
40
+ growing_steps = sum(1 for a, b in pairwise(inputs) if b >= a)
41
+ if growing_steps < _MONOTONIC_FRACTION * (len(inputs) - 1):
42
+ return []
43
+ if inputs[-1] < _GROWTH_FACTOR * inputs[0]:
44
+ return []
45
+ total_input = sum(inputs)
46
+ total_read = sum(r.usage.cache_read or 0 for r in requests)
47
+ if total_input == 0 or total_read / total_input > _MAX_READ_RATIO:
48
+ return []
49
+
50
+ # Waste from usage alone: each call re-sends roughly the prior call's
51
+ # content; whatever wasn't served from cache was re-billed.
52
+ resent = [
53
+ max(min(b, a) - read, 0)
54
+ for (a, b), read in zip(
55
+ pairwise(inputs), [r.usage.cache_read or 0 for r in requests[1:]], strict=True
56
+ )
57
+ ]
58
+ confirmed = [
59
+ amount for (a, b), amount in zip(pairwise(inputs), resent, strict=True) if b >= a
60
+ ]
61
+ tokens_high = round(0.9 * sum(resent))
62
+ tokens_low = round(0.9 * sum(confirmed))
63
+
64
+ finding = Finding(
65
+ detector_id=self.detector_id,
66
+ severity=Severity.HIGH,
67
+ session_id=session.session_id,
68
+ source=requests[0].source,
69
+ span_ids=tuple(r.span_id for r in requests),
70
+ summary=(
71
+ f"input grows from {inputs[0]:,} to {inputs[-1]:,} tokens across "
72
+ f"{len(requests)} calls with a {total_read / total_input:.0%} cache read "
73
+ "ratio — the conversation is re-sent at full price on every call"
74
+ ),
75
+ evidence={
76
+ "input_first": inputs[0],
77
+ "input_last": inputs[-1],
78
+ "growing_steps": growing_steps,
79
+ "steps": len(inputs) - 1,
80
+ "cache_read_ratio": round(total_read / total_input, 4),
81
+ "cache_markers_present": False,
82
+ },
83
+ impact=Impact(
84
+ tokens_low=tokens_low,
85
+ tokens_high=tokens_high,
86
+ usd_low=None,
87
+ usd_high=None,
88
+ assumption=(
89
+ "each call re-sends the prior call's content (monotonic growth pattern); "
90
+ "amounts come from reported usage, not estimates; 0.9x cached-rate delta; "
91
+ "low bound counts only confirmed growth steps"
92
+ ),
93
+ rulebook_version=rulebook.version,
94
+ ),
95
+ fix=Fix(
96
+ summary=(
97
+ "Cache the stable prefix (system prompt + history) with cache markers or "
98
+ "automatic prefix caching, and/or truncate or summarize history instead "
99
+ "of re-sending all of it every call."
100
+ ),
101
+ hint=None,
102
+ doc_url=None,
103
+ ),
104
+ )
105
+ return [finding]
@@ -0,0 +1,120 @@
1
+ """Lookback overflow: the reusable prefix sits too far behind the breakpoint.
2
+
3
+ A breakpoint checks a bounded number of positions (20 on Anthropic,
4
+ counting itself first). When history grows by more blocks than that between
5
+ calls, the previously cached prefix is out of reach and never read — even
6
+ though it is byte-identical and still alive.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from tokenaudit.detect.base import (
12
+ TOKEN_ESTIMATE_NOTE,
13
+ Abstain,
14
+ Detector,
15
+ Pair,
16
+ blocks_tokens,
17
+ consecutive_pairs,
18
+ )
19
+ from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
20
+ from tokenaudit.model.records import Session
21
+ from tokenaudit.rulebook import Rulebook
22
+
23
+
24
+ def _shared_prefix_len(pair: Pair) -> int:
25
+ a = pair.first.blocks or ()
26
+ b = pair.second.blocks or ()
27
+ shared = 0
28
+ for i in range(min(len(a), len(b))):
29
+ if a[i].sha256 != b[i].sha256:
30
+ break
31
+ shared = i + 1
32
+ return shared
33
+
34
+
35
+ class LookbackOverflowDetector(Detector):
36
+ detector_id = "lookback_overflow"
37
+
38
+ def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
39
+ pairs = consecutive_pairs(session)
40
+ if not pairs:
41
+ raise Abstain("needs consecutive same-model requests with captured content")
42
+
43
+ events: list[tuple[Pair, int, int, int]] = [] # (pair, marker idx, distance, lookback)
44
+ for pair in pairs:
45
+ blocks = pair.second.blocks or ()
46
+ marked = [j for j, block in enumerate(blocks) if block.cache_marker]
47
+ if not marked:
48
+ continue
49
+ rules = rulebook.lookup(pair.second.model, pair.second.provider)
50
+ if rules is None or rules.lookback_positions is None:
51
+ continue
52
+ shared = _shared_prefix_len(pair)
53
+ if shared == 0:
54
+ continue
55
+ marker = marked[-1]
56
+ distance = marker - (shared - 1)
57
+ shared_tokens = blocks_tokens(blocks[:shared])
58
+ read = pair.second.usage.cache_read
59
+ # Miss condition: reads cover less than half of the identical,
60
+ # still-live prefix - the reusable region is provably out of reach.
61
+ missed = read is not None and read < 0.5 * shared_tokens
62
+ if distance >= rules.lookback_positions and missed:
63
+ events.append((pair, marker, distance, rules.lookback_positions))
64
+ if not events:
65
+ return []
66
+
67
+ per_event = []
68
+ for pair, _, _, _ in events:
69
+ shared_tokens = blocks_tokens((pair.second.blocks or ())[: _shared_prefix_len(pair)])
70
+ per_event.append(max(shared_tokens - (pair.second.usage.cache_read or 0), 0))
71
+ tokens_high = round(0.9 * sum(per_event))
72
+ tokens_low = tokens_high # every event already required an observed shortfall
73
+
74
+ pair, marker, distance, lookback = events[0]
75
+ rules = rulebook.lookup(pair.second.model, pair.second.provider)
76
+ finding = Finding(
77
+ detector_id=self.detector_id,
78
+ severity=Severity.MEDIUM,
79
+ session_id=session.session_id,
80
+ source=pair.second.source,
81
+ span_ids=tuple(p.second.span_id for p, _, _, _ in events),
82
+ summary=(
83
+ f"history grows so fast that the cached prefix sits {distance} positions "
84
+ f"behind the breakpoint — beyond the {lookback}-position lookback — so an "
85
+ f"identical, still-live prefix is never reused ({len(events)} event(s))"
86
+ ),
87
+ evidence={
88
+ "events": [
89
+ {
90
+ "span_id": p.second.span_id,
91
+ "marker_index": m,
92
+ "prefix_distance": d,
93
+ "lookback_positions": lb,
94
+ }
95
+ for p, m, d, lb in events
96
+ ],
97
+ },
98
+ impact=Impact(
99
+ tokens_low=tokens_low,
100
+ tokens_high=tokens_high,
101
+ usd_low=None,
102
+ usd_high=None,
103
+ assumption=(
104
+ "the out-of-reach identical prefix would have been read at the cached "
105
+ f"rate (0.9x delta); {TOKEN_ESTIMATE_NOTE}; low bound counts events "
106
+ "with an observed zero cache read"
107
+ ),
108
+ rulebook_version=rulebook.version,
109
+ ),
110
+ fix=Fix(
111
+ summary=(
112
+ "Add intermediate cache markers as history grows (a moving breakpoint "
113
+ "every few turns) so each new breakpoint finds the previous one inside "
114
+ "its lookback window."
115
+ ),
116
+ hint=f"keep marker spacing under {lookback} blocks",
117
+ doc_url=rules.source_url if rules else None,
118
+ ),
119
+ )
120
+ return [finding]
@@ -0,0 +1,107 @@
1
+ """Model switch: an identical warm prefix thrown away at a model boundary.
2
+
3
+ Provider caches are keyed per model, so switching models mid-session
4
+ discards every cached prefix even when the prompt is byte-identical. Fires
5
+ only when the blocks on both sides of the boundary actually share the
6
+ cacheable prefix — a switch accompanied by a different prompt didn't lose
7
+ anything to the switch itself."""
8
+
9
+ from __future__ import annotations
10
+
11
+ from itertools import pairwise
12
+
13
+ from tokenaudit.detect.base import (
14
+ TOKEN_ESTIMATE_NOTE,
15
+ Abstain,
16
+ Detector,
17
+ blocks_tokens,
18
+ last_marked_index,
19
+ require_content,
20
+ )
21
+ from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
22
+ from tokenaudit.model.records import Request, Session
23
+ from tokenaudit.rulebook import Rulebook
24
+
25
+
26
+ def _shared_prefix_tokens(a: Request, b: Request) -> int:
27
+ """Tokens of the identical cacheable prefix across the boundary."""
28
+ a_blocks = a.blocks or ()
29
+ b_blocks = b.blocks or ()
30
+ shared = 0
31
+ for i in range(min(len(a_blocks), len(b_blocks))):
32
+ if a_blocks[i].sha256 != b_blocks[i].sha256:
33
+ break
34
+ shared = i + 1
35
+ if shared == 0:
36
+ return 0
37
+ end = last_marked_index(b_blocks)
38
+ limit = shared if end is None else min(shared, end + 1)
39
+ return blocks_tokens(b_blocks[:limit])
40
+
41
+
42
+ class ModelSwitchDetector(Detector):
43
+ detector_id = "model_switch"
44
+
45
+ def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
46
+ requests = require_content(session, minimum_requests=2)
47
+ boundaries: list[tuple[Request, Request, int]] = []
48
+ for a, b in pairwise(requests):
49
+ if a.model and b.model and a.model != b.model:
50
+ shared = _shared_prefix_tokens(a, b)
51
+ if shared > 0:
52
+ boundaries.append((a, b, shared))
53
+ if not boundaries:
54
+ return []
55
+ if len({r.model for r in requests}) > 2 or len(boundaries) > 2:
56
+ # Frequent alternation looks like deliberate model tiering; the
57
+ # fix differs (cache per tier), so don't blame "a switch".
58
+ raise Abstain(
59
+ "more than two models or repeated alternation: looks like deliberate "
60
+ "tiering, which needs per-tier analysis rather than a switch finding"
61
+ )
62
+
63
+ findings = []
64
+ for a, b, shared in boundaries:
65
+ observed_rewrite = (b.usage.cache_creation or 0) > 0 or b.usage.cache_read == 0
66
+ rules = rulebook.lookup(b.model, b.provider)
67
+ findings.append(
68
+ Finding(
69
+ detector_id=self.detector_id,
70
+ severity=Severity.MEDIUM,
71
+ session_id=session.session_id,
72
+ source=b.source,
73
+ span_ids=(a.span_id, b.span_id),
74
+ summary=(
75
+ f"the session switches from {a.model} to {b.model} with an identical "
76
+ f"~{shared}-token prefix; caches are per-model, so the warm prefix is "
77
+ "discarded and rebuilt"
78
+ ),
79
+ evidence={
80
+ "from_model": a.model,
81
+ "to_model": b.model,
82
+ "shared_prefix_est_tokens": shared,
83
+ "observed_rewrite_after_switch": observed_rewrite,
84
+ },
85
+ impact=Impact(
86
+ tokens_low=shared if observed_rewrite else 0,
87
+ tokens_high=shared,
88
+ usd_low=None,
89
+ usd_high=None,
90
+ assumption=(
91
+ "the shared prefix is re-billed once at the new model instead of "
92
+ f"read from cache; {TOKEN_ESTIMATE_NOTE}; low bound requires an "
93
+ "observed rewrite after the switch"
94
+ ),
95
+ rulebook_version=rulebook.version,
96
+ ),
97
+ fix=Fix(
98
+ summary=(
99
+ "Pin the session to one model, or accept one rebuild per deliberate "
100
+ "switch and keep switches out of hot loops."
101
+ ),
102
+ hint=None,
103
+ doc_url=rules.source_url if rules else None,
104
+ ),
105
+ )
106
+ )
107
+ return findings
@@ -0,0 +1,157 @@
1
+ """Prefix drift: the cacheable start of the prompt keeps changing.
2
+
3
+ The classic silent killer — a timestamp, counter, or mutating tracker
4
+ embedded early in the prompt makes the prefix miss byte-identical matching
5
+ on every call, so the session pays cache writes (or full price) forever.
6
+
7
+ Three guards keep this detector from crying wolf, each earned against a
8
+ labeled negative case:
9
+
10
+ - **Trailing-block exclusion**: the last block changing is just the new
11
+ user turn (every healthy conversation does it). A divergence only counts
12
+ when it sits before the final block or on a cache-marked block.
13
+ - **Size-similarity guard**: drift means the *same* block mutated slightly
14
+ (equal or near-equal length, different hash). A block replaced by
15
+ something of a very different size is a different prompt, not drift —
16
+ which is what consecutive calls from different agent roles look like in
17
+ real multi-agent traces.
18
+ - **Same-index majority**: the divergence must recur at the same block
19
+ index in at least half of the compared pairs (and at least twice). A
20
+ one-off change — e.g. tool definitions reordered once — is attributed by
21
+ a different detector; ground truth in the corpus encodes that precedence.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from tokenaudit.detect.base import (
27
+ TOKEN_ESTIMATE_NOTE,
28
+ Abstain,
29
+ Detector,
30
+ Pair,
31
+ blocks_tokens,
32
+ consecutive_pairs,
33
+ last_marked_index,
34
+ )
35
+ from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
36
+ from tokenaudit.model.records import Session
37
+ from tokenaudit.rulebook import Rulebook
38
+
39
+ _SIMILAR_ABS = 32 # chars
40
+ _SIMILAR_FRAC = 0.02
41
+ _TURN_ROLES = {"user", "human", "tool"} # roles that legitimately change every call
42
+
43
+
44
+ def _similar_size(a_chars: int, b_chars: int) -> bool:
45
+ return abs(a_chars - b_chars) <= max(_SIMILAR_ABS, int(_SIMILAR_FRAC * max(a_chars, b_chars)))
46
+
47
+
48
+ def _qualifying_divergence(pair: Pair) -> int | None:
49
+ d = pair.first_divergence
50
+ if d is None:
51
+ return None
52
+ a = pair.first.blocks or ()
53
+ b = pair.second.blocks or ()
54
+ trailing = d >= min(len(a), len(b)) - 1
55
+ if trailing and not a[d].cache_marker and a[d].role in _TURN_ROLES:
56
+ return None # the new user turn changing is not drift; a lone mutating
57
+ # system prompt (single-block prompts exist in legacy traces) still is
58
+ if a[d].role != b[d].role or not _similar_size(a[d].chars, b[d].chars):
59
+ return None # replaced content, not a mutated block
60
+ return d
61
+
62
+
63
+ class PrefixDriftDetector(Detector):
64
+ detector_id = "prefix_drift"
65
+
66
+ def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
67
+ pairs = consecutive_pairs(session)
68
+ if len(pairs) < 2:
69
+ raise Abstain("needs at least 3 same-model requests with captured content")
70
+
71
+ by_index: dict[int, list[Pair]] = {}
72
+ for pair in pairs:
73
+ d = _qualifying_divergence(pair)
74
+ if d is not None:
75
+ by_index.setdefault(d, []).append(pair)
76
+ if not by_index:
77
+ return []
78
+
79
+ index, affected = max(by_index.items(), key=lambda item: len(item[1]))
80
+ if len(affected) < 2 or len(affected) * 2 < len(pairs):
81
+ return [] # no recurring same-index drift; one-off changes belong elsewhere
82
+
83
+ # Counterfactual, one fix at a time: with a stable prefix, everything
84
+ # from the divergence point through the cacheable region would have
85
+ # been read at the cached rate instead of re-billed.
86
+ per_pair: list[int] = []
87
+ observed_misses = 0
88
+ for pair in affected:
89
+ blocks = pair.second.blocks or ()
90
+ end = last_marked_index(blocks)
91
+ end = end if end is not None and end >= index else len(blocks) - 2
92
+ per_pair.append(blocks_tokens(blocks[index : end + 1]))
93
+ usage = pair.second.usage
94
+ if usage.cache_read == 0 or (
95
+ usage.cache_creation is not None and usage.cache_creation > 0
96
+ ):
97
+ observed_misses += 1
98
+ tokens_high = round(0.9 * sum(per_pair))
99
+ tokens_low = round(0.9 * sum(sorted(per_pair)[:observed_misses]))
100
+
101
+ example = affected[0]
102
+ a = example.first.blocks or ()
103
+ b = example.second.blocks or ()
104
+ drift_kind = (
105
+ "same_length_mutation" if a[index].chars == b[index].chars else "small_mutation"
106
+ )
107
+ rules = rulebook.lookup(example.second.model, example.second.provider)
108
+
109
+ finding = Finding(
110
+ detector_id=self.detector_id,
111
+ severity=Severity.HIGH,
112
+ session_id=session.session_id,
113
+ source=example.second.source,
114
+ span_ids=tuple(p.second.span_id for p in affected),
115
+ summary=(
116
+ f"the prompt prefix changes at block {index} in {len(affected)} of "
117
+ f"{len(pairs)} consecutive calls, so the cacheable prefix never matches"
118
+ ),
119
+ evidence={
120
+ "block_index": index,
121
+ "block_role": a[index].role,
122
+ "block_chars": a[index].chars,
123
+ "drift_kind": drift_kind,
124
+ "pairs_affected": len(affected),
125
+ "pairs_compared": len(pairs),
126
+ "observed_cache_misses": observed_misses,
127
+ "example_before_sha256": a[index].sha256,
128
+ "example_after_sha256": b[index].sha256,
129
+ "example_span_ids": [example.first.span_id, example.second.span_id],
130
+ },
131
+ impact=Impact(
132
+ tokens_low=tokens_low,
133
+ tokens_high=tokens_high,
134
+ usd_low=None,
135
+ usd_high=None,
136
+ assumption=(
137
+ "difference between full-price and cached-read rates (0.9x) on the "
138
+ f"invalidated prefix region; {TOKEN_ESTIMATE_NOTE}; low bound counts "
139
+ "only pairs with an observed cache miss"
140
+ ),
141
+ rulebook_version=rulebook.version,
142
+ ),
143
+ fix=Fix(
144
+ summary=(
145
+ "Move the changing value out of the cached prefix — put timestamps, "
146
+ "counters and per-call state in the user message, not the system prompt."
147
+ ),
148
+ hint=(
149
+ "the mutating block keeps its length, which is what an embedded "
150
+ "timestamp or counter looks like"
151
+ if drift_kind == "same_length_mutation"
152
+ else "the mutating block changes size slightly on each call"
153
+ ),
154
+ doc_url=rules.source_url if rules else None,
155
+ ),
156
+ )
157
+ return [finding]
@@ -0,0 +1,102 @@
1
+ """Retry storm: byte-identical requests billed repeatedly within a short window.
2
+
3
+ Client timeouts and non-idempotent re-asks complete server-side and bill in
4
+ full; the trace shows the same prompt (identical block hash chain) landing
5
+ several times within seconds. Impact comes straight from reported usage —
6
+ no estimation — because every duplicate's billed input beyond its cache
7
+ reads is waste by definition, whatever the caching setup.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from tokenaudit.detect.base import Detector, require_content
13
+ from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
14
+ from tokenaudit.model.records import Request, Session
15
+ from tokenaudit.rulebook import Rulebook
16
+
17
+ _MIN_DUPLICATES = 3 # identical requests in a burst (1 original + 2 repeats)
18
+ _WINDOW_S = 120
19
+
20
+
21
+ def _chain(request: Request) -> tuple[str, ...]:
22
+ return tuple(block.sha256 for block in request.blocks or ())
23
+
24
+
25
+ class RetryStormDetector(Detector):
26
+ detector_id = "retry_storm"
27
+
28
+ def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
29
+ requests = require_content(session, minimum_requests=_MIN_DUPLICATES)
30
+
31
+ bursts: list[list[Request]] = []
32
+ current: list[Request] = []
33
+ for request in requests:
34
+ if (
35
+ current
36
+ and _chain(request) == _chain(current[-1])
37
+ and request.model == current[-1].model
38
+ and (request.start_unix_nano - current[-1].start_unix_nano)
39
+ <= _WINDOW_S * 1_000_000_000
40
+ ):
41
+ current.append(request)
42
+ else:
43
+ if len(current) >= _MIN_DUPLICATES:
44
+ bursts.append(current)
45
+ current = [request]
46
+ if len(current) >= _MIN_DUPLICATES:
47
+ bursts.append(current)
48
+ if not bursts:
49
+ return []
50
+
51
+ findings = []
52
+ for burst in bursts:
53
+ duplicates = burst[1:]
54
+ wasted_input = sum(
55
+ max((r.usage.input_total or 0) - (r.usage.cache_read or 0), 0) for r in duplicates
56
+ )
57
+ wasted_output = sum(r.usage.output or 0 for r in duplicates)
58
+ window_s = (burst[-1].start_unix_nano - burst[0].start_unix_nano) // 1_000_000_000
59
+ findings.append(
60
+ Finding(
61
+ detector_id=self.detector_id,
62
+ severity=Severity.HIGH,
63
+ session_id=session.session_id,
64
+ source=burst[0].source,
65
+ span_ids=tuple(r.span_id for r in burst),
66
+ summary=(
67
+ f"{len(burst)} byte-identical requests completed within {window_s}s — "
68
+ f"{len(duplicates)} duplicate(s) billed in full, the signature of "
69
+ "client-side retries against calls that succeeded server-side"
70
+ ),
71
+ evidence={
72
+ "burst_size": len(burst),
73
+ "window_seconds": int(window_s),
74
+ "duplicate_span_ids": [r.span_id for r in duplicates],
75
+ "wasted_output_tokens": wasted_output,
76
+ "prompt_chain_sha256": _chain(burst[0])[0],
77
+ },
78
+ impact=Impact(
79
+ tokens_low=wasted_input,
80
+ tokens_high=wasted_input,
81
+ usd_low=None,
82
+ usd_high=None,
83
+ assumption=(
84
+ "duplicates of an identical prompt within the burst window are "
85
+ "retries; billed input beyond cache reads comes from reported "
86
+ "usage, no estimation (duplicate output tokens reported separately "
87
+ "in evidence)"
88
+ ),
89
+ rulebook_version=rulebook.version,
90
+ ),
91
+ fix=Fix(
92
+ summary=(
93
+ "Make retries idempotent: raise client timeouts above realistic "
94
+ "completion times, use request ids / idempotency keys where the "
95
+ "provider supports them, and back off instead of re-firing."
96
+ ),
97
+ hint=None,
98
+ doc_url=None,
99
+ ),
100
+ )
101
+ )
102
+ return findings