tokenaudit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokenaudit/__init__.py +3 -0
- tokenaudit/__main__.py +152 -0
- tokenaudit/ci/__init__.py +1 -0
- tokenaudit/counterfactual/__init__.py +1 -0
- tokenaudit/detect/__init__.py +15 -0
- tokenaudit/detect/base.py +96 -0
- tokenaudit/detect/corpus_record.json +56 -0
- tokenaudit/detect/engine.py +87 -0
- tokenaudit/detect/findings.py +86 -0
- tokenaudit/detect/rules/__init__.py +1 -0
- tokenaudit/detect/rules/breakpoint_limit.py +102 -0
- tokenaudit/detect/rules/context_bloat.py +105 -0
- tokenaudit/detect/rules/lookback_overflow.py +120 -0
- tokenaudit/detect/rules/model_switch.py +107 -0
- tokenaudit/detect/rules/prefix_drift.py +157 -0
- tokenaudit/detect/rules/retry_storm.py +102 -0
- tokenaudit/detect/rules/sub_minimum.py +112 -0
- tokenaudit/detect/rules/tool_schema_churn.py +120 -0
- tokenaudit/detect/rules/ttl_expiry.py +121 -0
- tokenaudit/examples/sample.trace.json +512 -0
- tokenaudit/ingest/__init__.py +39 -0
- tokenaudit/ingest/containers.py +172 -0
- tokenaudit/ingest/dedup.py +64 -0
- tokenaudit/ingest/dialects.py +258 -0
- tokenaudit/ingest/quality.py +65 -0
- tokenaudit/ingest/sessionize.py +47 -0
- tokenaudit/model/__init__.py +5 -0
- tokenaudit/model/records.py +99 -0
- tokenaudit/py.typed +0 -0
- tokenaudit/report/__init__.py +14 -0
- tokenaudit/report/corpus_record.py +35 -0
- tokenaudit/report/health.py +86 -0
- tokenaudit/report/html.py +152 -0
- tokenaudit/rulebook/__init__.py +238 -0
- tokenaudit/rulebook/anthropic.json +36 -0
- tokenaudit/rulebook/google.json +23 -0
- tokenaudit/rulebook/openai.json +45 -0
- tokenaudit-0.1.0.dist-info/METADATA +111 -0
- tokenaudit-0.1.0.dist-info/RECORD +42 -0
- tokenaudit-0.1.0.dist-info/WHEEL +4 -0
- tokenaudit-0.1.0.dist-info/entry_points.txt +2 -0
- tokenaudit-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Context bloat: the whole conversation re-sent every call, uncached.
|
|
2
|
+
|
|
3
|
+
The only usage-driven detector in the library: it needs no message content,
|
|
4
|
+
so it still works on traces captured with content capture off (the OTel
|
|
5
|
+
default). Fires when input grows monotonically across an unmanaged session
|
|
6
|
+
— no cache markers anywhere — while cache reads stay near zero, i.e. every
|
|
7
|
+
repeated token is billed at full price, forever, by construction.
|
|
8
|
+
|
|
9
|
+
Sessions that use cache markers are excluded: growth under markers has
|
|
10
|
+
specific failure modes (lookback overflow, breakpoint budget) with their
|
|
11
|
+
own detectors, and blaming "bloat" there would suggest the wrong fix.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from itertools import pairwise
|
|
17
|
+
|
|
18
|
+
from tokenaudit.detect.base import Abstain, Detector
|
|
19
|
+
from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
|
|
20
|
+
from tokenaudit.model.records import Session
|
|
21
|
+
from tokenaudit.rulebook import Rulebook
|
|
22
|
+
|
|
23
|
+
_MIN_REQUESTS = 5
|
|
24
|
+
_MONOTONIC_FRACTION = 0.8
|
|
25
|
+
_GROWTH_FACTOR = 2.0
|
|
26
|
+
_MAX_READ_RATIO = 0.3
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ContextBloatDetector(Detector):
|
|
30
|
+
detector_id = "context_bloat"
|
|
31
|
+
|
|
32
|
+
def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
|
|
33
|
+
requests = [r for r in session.requests if r.usage.input_total is not None]
|
|
34
|
+
if len(requests) < _MIN_REQUESTS:
|
|
35
|
+
raise Abstain(f"needs at least {_MIN_REQUESTS} requests with input token counts")
|
|
36
|
+
if any(block.cache_marker for r in requests if r.blocks for block in r.blocks):
|
|
37
|
+
return [] # marker-managed growth has its own, more specific detectors
|
|
38
|
+
|
|
39
|
+
inputs = [r.usage.input_total or 0 for r in requests]
|
|
40
|
+
growing_steps = sum(1 for a, b in pairwise(inputs) if b >= a)
|
|
41
|
+
if growing_steps < _MONOTONIC_FRACTION * (len(inputs) - 1):
|
|
42
|
+
return []
|
|
43
|
+
if inputs[-1] < _GROWTH_FACTOR * inputs[0]:
|
|
44
|
+
return []
|
|
45
|
+
total_input = sum(inputs)
|
|
46
|
+
total_read = sum(r.usage.cache_read or 0 for r in requests)
|
|
47
|
+
if total_input == 0 or total_read / total_input > _MAX_READ_RATIO:
|
|
48
|
+
return []
|
|
49
|
+
|
|
50
|
+
# Waste from usage alone: each call re-sends roughly the prior call's
|
|
51
|
+
# content; whatever wasn't served from cache was re-billed.
|
|
52
|
+
resent = [
|
|
53
|
+
max(min(b, a) - read, 0)
|
|
54
|
+
for (a, b), read in zip(
|
|
55
|
+
pairwise(inputs), [r.usage.cache_read or 0 for r in requests[1:]], strict=True
|
|
56
|
+
)
|
|
57
|
+
]
|
|
58
|
+
confirmed = [
|
|
59
|
+
amount for (a, b), amount in zip(pairwise(inputs), resent, strict=True) if b >= a
|
|
60
|
+
]
|
|
61
|
+
tokens_high = round(0.9 * sum(resent))
|
|
62
|
+
tokens_low = round(0.9 * sum(confirmed))
|
|
63
|
+
|
|
64
|
+
finding = Finding(
|
|
65
|
+
detector_id=self.detector_id,
|
|
66
|
+
severity=Severity.HIGH,
|
|
67
|
+
session_id=session.session_id,
|
|
68
|
+
source=requests[0].source,
|
|
69
|
+
span_ids=tuple(r.span_id for r in requests),
|
|
70
|
+
summary=(
|
|
71
|
+
f"input grows from {inputs[0]:,} to {inputs[-1]:,} tokens across "
|
|
72
|
+
f"{len(requests)} calls with a {total_read / total_input:.0%} cache read "
|
|
73
|
+
"ratio — the conversation is re-sent at full price on every call"
|
|
74
|
+
),
|
|
75
|
+
evidence={
|
|
76
|
+
"input_first": inputs[0],
|
|
77
|
+
"input_last": inputs[-1],
|
|
78
|
+
"growing_steps": growing_steps,
|
|
79
|
+
"steps": len(inputs) - 1,
|
|
80
|
+
"cache_read_ratio": round(total_read / total_input, 4),
|
|
81
|
+
"cache_markers_present": False,
|
|
82
|
+
},
|
|
83
|
+
impact=Impact(
|
|
84
|
+
tokens_low=tokens_low,
|
|
85
|
+
tokens_high=tokens_high,
|
|
86
|
+
usd_low=None,
|
|
87
|
+
usd_high=None,
|
|
88
|
+
assumption=(
|
|
89
|
+
"each call re-sends the prior call's content (monotonic growth pattern); "
|
|
90
|
+
"amounts come from reported usage, not estimates; 0.9x cached-rate delta; "
|
|
91
|
+
"low bound counts only confirmed growth steps"
|
|
92
|
+
),
|
|
93
|
+
rulebook_version=rulebook.version,
|
|
94
|
+
),
|
|
95
|
+
fix=Fix(
|
|
96
|
+
summary=(
|
|
97
|
+
"Cache the stable prefix (system prompt + history) with cache markers or "
|
|
98
|
+
"automatic prefix caching, and/or truncate or summarize history instead "
|
|
99
|
+
"of re-sending all of it every call."
|
|
100
|
+
),
|
|
101
|
+
hint=None,
|
|
102
|
+
doc_url=None,
|
|
103
|
+
),
|
|
104
|
+
)
|
|
105
|
+
return [finding]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Lookback overflow: the reusable prefix sits too far behind the breakpoint.
|
|
2
|
+
|
|
3
|
+
A breakpoint checks a bounded number of positions (20 on Anthropic,
|
|
4
|
+
counting itself first). When history grows by more blocks than that between
|
|
5
|
+
calls, the previously cached prefix is out of reach and never read — even
|
|
6
|
+
though it is byte-identical and still alive.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from tokenaudit.detect.base import (
|
|
12
|
+
TOKEN_ESTIMATE_NOTE,
|
|
13
|
+
Abstain,
|
|
14
|
+
Detector,
|
|
15
|
+
Pair,
|
|
16
|
+
blocks_tokens,
|
|
17
|
+
consecutive_pairs,
|
|
18
|
+
)
|
|
19
|
+
from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
|
|
20
|
+
from tokenaudit.model.records import Session
|
|
21
|
+
from tokenaudit.rulebook import Rulebook
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _shared_prefix_len(pair: Pair) -> int:
|
|
25
|
+
a = pair.first.blocks or ()
|
|
26
|
+
b = pair.second.blocks or ()
|
|
27
|
+
shared = 0
|
|
28
|
+
for i in range(min(len(a), len(b))):
|
|
29
|
+
if a[i].sha256 != b[i].sha256:
|
|
30
|
+
break
|
|
31
|
+
shared = i + 1
|
|
32
|
+
return shared
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LookbackOverflowDetector(Detector):
|
|
36
|
+
detector_id = "lookback_overflow"
|
|
37
|
+
|
|
38
|
+
def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
|
|
39
|
+
pairs = consecutive_pairs(session)
|
|
40
|
+
if not pairs:
|
|
41
|
+
raise Abstain("needs consecutive same-model requests with captured content")
|
|
42
|
+
|
|
43
|
+
events: list[tuple[Pair, int, int, int]] = [] # (pair, marker idx, distance, lookback)
|
|
44
|
+
for pair in pairs:
|
|
45
|
+
blocks = pair.second.blocks or ()
|
|
46
|
+
marked = [j for j, block in enumerate(blocks) if block.cache_marker]
|
|
47
|
+
if not marked:
|
|
48
|
+
continue
|
|
49
|
+
rules = rulebook.lookup(pair.second.model, pair.second.provider)
|
|
50
|
+
if rules is None or rules.lookback_positions is None:
|
|
51
|
+
continue
|
|
52
|
+
shared = _shared_prefix_len(pair)
|
|
53
|
+
if shared == 0:
|
|
54
|
+
continue
|
|
55
|
+
marker = marked[-1]
|
|
56
|
+
distance = marker - (shared - 1)
|
|
57
|
+
shared_tokens = blocks_tokens(blocks[:shared])
|
|
58
|
+
read = pair.second.usage.cache_read
|
|
59
|
+
# Miss condition: reads cover less than half of the identical,
|
|
60
|
+
# still-live prefix - the reusable region is provably out of reach.
|
|
61
|
+
missed = read is not None and read < 0.5 * shared_tokens
|
|
62
|
+
if distance >= rules.lookback_positions and missed:
|
|
63
|
+
events.append((pair, marker, distance, rules.lookback_positions))
|
|
64
|
+
if not events:
|
|
65
|
+
return []
|
|
66
|
+
|
|
67
|
+
per_event = []
|
|
68
|
+
for pair, _, _, _ in events:
|
|
69
|
+
shared_tokens = blocks_tokens((pair.second.blocks or ())[: _shared_prefix_len(pair)])
|
|
70
|
+
per_event.append(max(shared_tokens - (pair.second.usage.cache_read or 0), 0))
|
|
71
|
+
tokens_high = round(0.9 * sum(per_event))
|
|
72
|
+
tokens_low = tokens_high # every event already required an observed shortfall
|
|
73
|
+
|
|
74
|
+
pair, marker, distance, lookback = events[0]
|
|
75
|
+
rules = rulebook.lookup(pair.second.model, pair.second.provider)
|
|
76
|
+
finding = Finding(
|
|
77
|
+
detector_id=self.detector_id,
|
|
78
|
+
severity=Severity.MEDIUM,
|
|
79
|
+
session_id=session.session_id,
|
|
80
|
+
source=pair.second.source,
|
|
81
|
+
span_ids=tuple(p.second.span_id for p, _, _, _ in events),
|
|
82
|
+
summary=(
|
|
83
|
+
f"history grows so fast that the cached prefix sits {distance} positions "
|
|
84
|
+
f"behind the breakpoint — beyond the {lookback}-position lookback — so an "
|
|
85
|
+
f"identical, still-live prefix is never reused ({len(events)} event(s))"
|
|
86
|
+
),
|
|
87
|
+
evidence={
|
|
88
|
+
"events": [
|
|
89
|
+
{
|
|
90
|
+
"span_id": p.second.span_id,
|
|
91
|
+
"marker_index": m,
|
|
92
|
+
"prefix_distance": d,
|
|
93
|
+
"lookback_positions": lb,
|
|
94
|
+
}
|
|
95
|
+
for p, m, d, lb in events
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
impact=Impact(
|
|
99
|
+
tokens_low=tokens_low,
|
|
100
|
+
tokens_high=tokens_high,
|
|
101
|
+
usd_low=None,
|
|
102
|
+
usd_high=None,
|
|
103
|
+
assumption=(
|
|
104
|
+
"the out-of-reach identical prefix would have been read at the cached "
|
|
105
|
+
f"rate (0.9x delta); {TOKEN_ESTIMATE_NOTE}; low bound counts events "
|
|
106
|
+
"with an observed zero cache read"
|
|
107
|
+
),
|
|
108
|
+
rulebook_version=rulebook.version,
|
|
109
|
+
),
|
|
110
|
+
fix=Fix(
|
|
111
|
+
summary=(
|
|
112
|
+
"Add intermediate cache markers as history grows (a moving breakpoint "
|
|
113
|
+
"every few turns) so each new breakpoint finds the previous one inside "
|
|
114
|
+
"its lookback window."
|
|
115
|
+
),
|
|
116
|
+
hint=f"keep marker spacing under {lookback} blocks",
|
|
117
|
+
doc_url=rules.source_url if rules else None,
|
|
118
|
+
),
|
|
119
|
+
)
|
|
120
|
+
return [finding]
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Model switch: an identical warm prefix thrown away at a model boundary.
|
|
2
|
+
|
|
3
|
+
Provider caches are keyed per model, so switching models mid-session
|
|
4
|
+
discards every cached prefix even when the prompt is byte-identical. Fires
|
|
5
|
+
only when the blocks on both sides of the boundary actually share the
|
|
6
|
+
cacheable prefix — a switch accompanied by a different prompt didn't lose
|
|
7
|
+
anything to the switch itself."""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from itertools import pairwise
|
|
12
|
+
|
|
13
|
+
from tokenaudit.detect.base import (
|
|
14
|
+
TOKEN_ESTIMATE_NOTE,
|
|
15
|
+
Abstain,
|
|
16
|
+
Detector,
|
|
17
|
+
blocks_tokens,
|
|
18
|
+
last_marked_index,
|
|
19
|
+
require_content,
|
|
20
|
+
)
|
|
21
|
+
from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
|
|
22
|
+
from tokenaudit.model.records import Request, Session
|
|
23
|
+
from tokenaudit.rulebook import Rulebook
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _shared_prefix_tokens(a: Request, b: Request) -> int:
|
|
27
|
+
"""Tokens of the identical cacheable prefix across the boundary."""
|
|
28
|
+
a_blocks = a.blocks or ()
|
|
29
|
+
b_blocks = b.blocks or ()
|
|
30
|
+
shared = 0
|
|
31
|
+
for i in range(min(len(a_blocks), len(b_blocks))):
|
|
32
|
+
if a_blocks[i].sha256 != b_blocks[i].sha256:
|
|
33
|
+
break
|
|
34
|
+
shared = i + 1
|
|
35
|
+
if shared == 0:
|
|
36
|
+
return 0
|
|
37
|
+
end = last_marked_index(b_blocks)
|
|
38
|
+
limit = shared if end is None else min(shared, end + 1)
|
|
39
|
+
return blocks_tokens(b_blocks[:limit])
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ModelSwitchDetector(Detector):
|
|
43
|
+
detector_id = "model_switch"
|
|
44
|
+
|
|
45
|
+
def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
|
|
46
|
+
requests = require_content(session, minimum_requests=2)
|
|
47
|
+
boundaries: list[tuple[Request, Request, int]] = []
|
|
48
|
+
for a, b in pairwise(requests):
|
|
49
|
+
if a.model and b.model and a.model != b.model:
|
|
50
|
+
shared = _shared_prefix_tokens(a, b)
|
|
51
|
+
if shared > 0:
|
|
52
|
+
boundaries.append((a, b, shared))
|
|
53
|
+
if not boundaries:
|
|
54
|
+
return []
|
|
55
|
+
if len({r.model for r in requests}) > 2 or len(boundaries) > 2:
|
|
56
|
+
# Frequent alternation looks like deliberate model tiering; the
|
|
57
|
+
# fix differs (cache per tier), so don't blame "a switch".
|
|
58
|
+
raise Abstain(
|
|
59
|
+
"more than two models or repeated alternation: looks like deliberate "
|
|
60
|
+
"tiering, which needs per-tier analysis rather than a switch finding"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
findings = []
|
|
64
|
+
for a, b, shared in boundaries:
|
|
65
|
+
observed_rewrite = (b.usage.cache_creation or 0) > 0 or b.usage.cache_read == 0
|
|
66
|
+
rules = rulebook.lookup(b.model, b.provider)
|
|
67
|
+
findings.append(
|
|
68
|
+
Finding(
|
|
69
|
+
detector_id=self.detector_id,
|
|
70
|
+
severity=Severity.MEDIUM,
|
|
71
|
+
session_id=session.session_id,
|
|
72
|
+
source=b.source,
|
|
73
|
+
span_ids=(a.span_id, b.span_id),
|
|
74
|
+
summary=(
|
|
75
|
+
f"the session switches from {a.model} to {b.model} with an identical "
|
|
76
|
+
f"~{shared}-token prefix; caches are per-model, so the warm prefix is "
|
|
77
|
+
"discarded and rebuilt"
|
|
78
|
+
),
|
|
79
|
+
evidence={
|
|
80
|
+
"from_model": a.model,
|
|
81
|
+
"to_model": b.model,
|
|
82
|
+
"shared_prefix_est_tokens": shared,
|
|
83
|
+
"observed_rewrite_after_switch": observed_rewrite,
|
|
84
|
+
},
|
|
85
|
+
impact=Impact(
|
|
86
|
+
tokens_low=shared if observed_rewrite else 0,
|
|
87
|
+
tokens_high=shared,
|
|
88
|
+
usd_low=None,
|
|
89
|
+
usd_high=None,
|
|
90
|
+
assumption=(
|
|
91
|
+
"the shared prefix is re-billed once at the new model instead of "
|
|
92
|
+
f"read from cache; {TOKEN_ESTIMATE_NOTE}; low bound requires an "
|
|
93
|
+
"observed rewrite after the switch"
|
|
94
|
+
),
|
|
95
|
+
rulebook_version=rulebook.version,
|
|
96
|
+
),
|
|
97
|
+
fix=Fix(
|
|
98
|
+
summary=(
|
|
99
|
+
"Pin the session to one model, or accept one rebuild per deliberate "
|
|
100
|
+
"switch and keep switches out of hot loops."
|
|
101
|
+
),
|
|
102
|
+
hint=None,
|
|
103
|
+
doc_url=rules.source_url if rules else None,
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
return findings
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Prefix drift: the cacheable start of the prompt keeps changing.
|
|
2
|
+
|
|
3
|
+
The classic silent killer — a timestamp, counter, or mutating tracker
|
|
4
|
+
embedded early in the prompt makes the prefix miss byte-identical matching
|
|
5
|
+
on every call, so the session pays cache writes (or full price) forever.
|
|
6
|
+
|
|
7
|
+
Three guards keep this detector from crying wolf, each earned against a
|
|
8
|
+
labeled negative case:
|
|
9
|
+
|
|
10
|
+
- **Trailing-block exclusion**: the last block changing is just the new
|
|
11
|
+
user turn (every healthy conversation does it). A divergence only counts
|
|
12
|
+
when it sits before the final block or on a cache-marked block.
|
|
13
|
+
- **Size-similarity guard**: drift means the *same* block mutated slightly
|
|
14
|
+
(equal or near-equal length, different hash). A block replaced by
|
|
15
|
+
something of a very different size is a different prompt, not drift —
|
|
16
|
+
which is what consecutive calls from different agent roles look like in
|
|
17
|
+
real multi-agent traces.
|
|
18
|
+
- **Same-index majority**: the divergence must recur at the same block
|
|
19
|
+
index in at least half of the compared pairs (and at least twice). A
|
|
20
|
+
one-off change — e.g. tool definitions reordered once — is attributed by
|
|
21
|
+
a different detector; ground truth in the corpus encodes that precedence.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from tokenaudit.detect.base import (
|
|
27
|
+
TOKEN_ESTIMATE_NOTE,
|
|
28
|
+
Abstain,
|
|
29
|
+
Detector,
|
|
30
|
+
Pair,
|
|
31
|
+
blocks_tokens,
|
|
32
|
+
consecutive_pairs,
|
|
33
|
+
last_marked_index,
|
|
34
|
+
)
|
|
35
|
+
from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
|
|
36
|
+
from tokenaudit.model.records import Session
|
|
37
|
+
from tokenaudit.rulebook import Rulebook
|
|
38
|
+
|
|
39
|
+
_SIMILAR_ABS = 32 # chars
|
|
40
|
+
_SIMILAR_FRAC = 0.02
|
|
41
|
+
_TURN_ROLES = {"user", "human", "tool"} # roles that legitimately change every call
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _similar_size(a_chars: int, b_chars: int) -> bool:
|
|
45
|
+
return abs(a_chars - b_chars) <= max(_SIMILAR_ABS, int(_SIMILAR_FRAC * max(a_chars, b_chars)))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _qualifying_divergence(pair: Pair) -> int | None:
|
|
49
|
+
d = pair.first_divergence
|
|
50
|
+
if d is None:
|
|
51
|
+
return None
|
|
52
|
+
a = pair.first.blocks or ()
|
|
53
|
+
b = pair.second.blocks or ()
|
|
54
|
+
trailing = d >= min(len(a), len(b)) - 1
|
|
55
|
+
if trailing and not a[d].cache_marker and a[d].role in _TURN_ROLES:
|
|
56
|
+
return None # the new user turn changing is not drift; a lone mutating
|
|
57
|
+
# system prompt (single-block prompts exist in legacy traces) still is
|
|
58
|
+
if a[d].role != b[d].role or not _similar_size(a[d].chars, b[d].chars):
|
|
59
|
+
return None # replaced content, not a mutated block
|
|
60
|
+
return d
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class PrefixDriftDetector(Detector):
|
|
64
|
+
detector_id = "prefix_drift"
|
|
65
|
+
|
|
66
|
+
def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
|
|
67
|
+
pairs = consecutive_pairs(session)
|
|
68
|
+
if len(pairs) < 2:
|
|
69
|
+
raise Abstain("needs at least 3 same-model requests with captured content")
|
|
70
|
+
|
|
71
|
+
by_index: dict[int, list[Pair]] = {}
|
|
72
|
+
for pair in pairs:
|
|
73
|
+
d = _qualifying_divergence(pair)
|
|
74
|
+
if d is not None:
|
|
75
|
+
by_index.setdefault(d, []).append(pair)
|
|
76
|
+
if not by_index:
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
index, affected = max(by_index.items(), key=lambda item: len(item[1]))
|
|
80
|
+
if len(affected) < 2 or len(affected) * 2 < len(pairs):
|
|
81
|
+
return [] # no recurring same-index drift; one-off changes belong elsewhere
|
|
82
|
+
|
|
83
|
+
# Counterfactual, one fix at a time: with a stable prefix, everything
|
|
84
|
+
# from the divergence point through the cacheable region would have
|
|
85
|
+
# been read at the cached rate instead of re-billed.
|
|
86
|
+
per_pair: list[int] = []
|
|
87
|
+
observed_misses = 0
|
|
88
|
+
for pair in affected:
|
|
89
|
+
blocks = pair.second.blocks or ()
|
|
90
|
+
end = last_marked_index(blocks)
|
|
91
|
+
end = end if end is not None and end >= index else len(blocks) - 2
|
|
92
|
+
per_pair.append(blocks_tokens(blocks[index : end + 1]))
|
|
93
|
+
usage = pair.second.usage
|
|
94
|
+
if usage.cache_read == 0 or (
|
|
95
|
+
usage.cache_creation is not None and usage.cache_creation > 0
|
|
96
|
+
):
|
|
97
|
+
observed_misses += 1
|
|
98
|
+
tokens_high = round(0.9 * sum(per_pair))
|
|
99
|
+
tokens_low = round(0.9 * sum(sorted(per_pair)[:observed_misses]))
|
|
100
|
+
|
|
101
|
+
example = affected[0]
|
|
102
|
+
a = example.first.blocks or ()
|
|
103
|
+
b = example.second.blocks or ()
|
|
104
|
+
drift_kind = (
|
|
105
|
+
"same_length_mutation" if a[index].chars == b[index].chars else "small_mutation"
|
|
106
|
+
)
|
|
107
|
+
rules = rulebook.lookup(example.second.model, example.second.provider)
|
|
108
|
+
|
|
109
|
+
finding = Finding(
|
|
110
|
+
detector_id=self.detector_id,
|
|
111
|
+
severity=Severity.HIGH,
|
|
112
|
+
session_id=session.session_id,
|
|
113
|
+
source=example.second.source,
|
|
114
|
+
span_ids=tuple(p.second.span_id for p in affected),
|
|
115
|
+
summary=(
|
|
116
|
+
f"the prompt prefix changes at block {index} in {len(affected)} of "
|
|
117
|
+
f"{len(pairs)} consecutive calls, so the cacheable prefix never matches"
|
|
118
|
+
),
|
|
119
|
+
evidence={
|
|
120
|
+
"block_index": index,
|
|
121
|
+
"block_role": a[index].role,
|
|
122
|
+
"block_chars": a[index].chars,
|
|
123
|
+
"drift_kind": drift_kind,
|
|
124
|
+
"pairs_affected": len(affected),
|
|
125
|
+
"pairs_compared": len(pairs),
|
|
126
|
+
"observed_cache_misses": observed_misses,
|
|
127
|
+
"example_before_sha256": a[index].sha256,
|
|
128
|
+
"example_after_sha256": b[index].sha256,
|
|
129
|
+
"example_span_ids": [example.first.span_id, example.second.span_id],
|
|
130
|
+
},
|
|
131
|
+
impact=Impact(
|
|
132
|
+
tokens_low=tokens_low,
|
|
133
|
+
tokens_high=tokens_high,
|
|
134
|
+
usd_low=None,
|
|
135
|
+
usd_high=None,
|
|
136
|
+
assumption=(
|
|
137
|
+
"difference between full-price and cached-read rates (0.9x) on the "
|
|
138
|
+
f"invalidated prefix region; {TOKEN_ESTIMATE_NOTE}; low bound counts "
|
|
139
|
+
"only pairs with an observed cache miss"
|
|
140
|
+
),
|
|
141
|
+
rulebook_version=rulebook.version,
|
|
142
|
+
),
|
|
143
|
+
fix=Fix(
|
|
144
|
+
summary=(
|
|
145
|
+
"Move the changing value out of the cached prefix — put timestamps, "
|
|
146
|
+
"counters and per-call state in the user message, not the system prompt."
|
|
147
|
+
),
|
|
148
|
+
hint=(
|
|
149
|
+
"the mutating block keeps its length, which is what an embedded "
|
|
150
|
+
"timestamp or counter looks like"
|
|
151
|
+
if drift_kind == "same_length_mutation"
|
|
152
|
+
else "the mutating block changes size slightly on each call"
|
|
153
|
+
),
|
|
154
|
+
doc_url=rules.source_url if rules else None,
|
|
155
|
+
),
|
|
156
|
+
)
|
|
157
|
+
return [finding]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Retry storm: byte-identical requests billed repeatedly within a short window.
|
|
2
|
+
|
|
3
|
+
Client timeouts and non-idempotent re-asks complete server-side and bill in
|
|
4
|
+
full; the trace shows the same prompt (identical block hash chain) landing
|
|
5
|
+
several times within seconds. Impact comes straight from reported usage —
|
|
6
|
+
no estimation — because every duplicate's billed input beyond its cache
|
|
7
|
+
reads is waste by definition, whatever the caching setup.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from tokenaudit.detect.base import Detector, require_content
|
|
13
|
+
from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
|
|
14
|
+
from tokenaudit.model.records import Request, Session
|
|
15
|
+
from tokenaudit.rulebook import Rulebook
|
|
16
|
+
|
|
17
|
+
_MIN_DUPLICATES = 3 # identical requests in a burst (1 original + 2 repeats)
|
|
18
|
+
_WINDOW_S = 120
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _chain(request: Request) -> tuple[str, ...]:
|
|
22
|
+
return tuple(block.sha256 for block in request.blocks or ())
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RetryStormDetector(Detector):
|
|
26
|
+
detector_id = "retry_storm"
|
|
27
|
+
|
|
28
|
+
def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
|
|
29
|
+
requests = require_content(session, minimum_requests=_MIN_DUPLICATES)
|
|
30
|
+
|
|
31
|
+
bursts: list[list[Request]] = []
|
|
32
|
+
current: list[Request] = []
|
|
33
|
+
for request in requests:
|
|
34
|
+
if (
|
|
35
|
+
current
|
|
36
|
+
and _chain(request) == _chain(current[-1])
|
|
37
|
+
and request.model == current[-1].model
|
|
38
|
+
and (request.start_unix_nano - current[-1].start_unix_nano)
|
|
39
|
+
<= _WINDOW_S * 1_000_000_000
|
|
40
|
+
):
|
|
41
|
+
current.append(request)
|
|
42
|
+
else:
|
|
43
|
+
if len(current) >= _MIN_DUPLICATES:
|
|
44
|
+
bursts.append(current)
|
|
45
|
+
current = [request]
|
|
46
|
+
if len(current) >= _MIN_DUPLICATES:
|
|
47
|
+
bursts.append(current)
|
|
48
|
+
if not bursts:
|
|
49
|
+
return []
|
|
50
|
+
|
|
51
|
+
findings = []
|
|
52
|
+
for burst in bursts:
|
|
53
|
+
duplicates = burst[1:]
|
|
54
|
+
wasted_input = sum(
|
|
55
|
+
max((r.usage.input_total or 0) - (r.usage.cache_read or 0), 0) for r in duplicates
|
|
56
|
+
)
|
|
57
|
+
wasted_output = sum(r.usage.output or 0 for r in duplicates)
|
|
58
|
+
window_s = (burst[-1].start_unix_nano - burst[0].start_unix_nano) // 1_000_000_000
|
|
59
|
+
findings.append(
|
|
60
|
+
Finding(
|
|
61
|
+
detector_id=self.detector_id,
|
|
62
|
+
severity=Severity.HIGH,
|
|
63
|
+
session_id=session.session_id,
|
|
64
|
+
source=burst[0].source,
|
|
65
|
+
span_ids=tuple(r.span_id for r in burst),
|
|
66
|
+
summary=(
|
|
67
|
+
f"{len(burst)} byte-identical requests completed within {window_s}s — "
|
|
68
|
+
f"{len(duplicates)} duplicate(s) billed in full, the signature of "
|
|
69
|
+
"client-side retries against calls that succeeded server-side"
|
|
70
|
+
),
|
|
71
|
+
evidence={
|
|
72
|
+
"burst_size": len(burst),
|
|
73
|
+
"window_seconds": int(window_s),
|
|
74
|
+
"duplicate_span_ids": [r.span_id for r in duplicates],
|
|
75
|
+
"wasted_output_tokens": wasted_output,
|
|
76
|
+
"prompt_chain_sha256": _chain(burst[0])[0],
|
|
77
|
+
},
|
|
78
|
+
impact=Impact(
|
|
79
|
+
tokens_low=wasted_input,
|
|
80
|
+
tokens_high=wasted_input,
|
|
81
|
+
usd_low=None,
|
|
82
|
+
usd_high=None,
|
|
83
|
+
assumption=(
|
|
84
|
+
"duplicates of an identical prompt within the burst window are "
|
|
85
|
+
"retries; billed input beyond cache reads comes from reported "
|
|
86
|
+
"usage, no estimation (duplicate output tokens reported separately "
|
|
87
|
+
"in evidence)"
|
|
88
|
+
),
|
|
89
|
+
rulebook_version=rulebook.version,
|
|
90
|
+
),
|
|
91
|
+
fix=Fix(
|
|
92
|
+
summary=(
|
|
93
|
+
"Make retries idempotent: raise client timeouts above realistic "
|
|
94
|
+
"completion times, use request ids / idempotency keys where the "
|
|
95
|
+
"provider supports them, and back off instead of re-firing."
|
|
96
|
+
),
|
|
97
|
+
hint=None,
|
|
98
|
+
doc_url=None,
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
return findings
|