prefixwise 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefixwise/__init__.py +10 -0
- prefixwise/__main__.py +3 -0
- prefixwise/classifier.py +283 -0
- prefixwise/cli.py +338 -0
- prefixwise/diagnostics.py +168 -0
- prefixwise/explain.py +155 -0
- prefixwise/fingerprint.py +159 -0
- prefixwise/fixes.py +105 -0
- prefixwise/ledger.py +294 -0
- prefixwise/prices.json +61 -0
- prefixwise/pricing.py +165 -0
- prefixwise/proxy.py +1075 -0
- prefixwise-0.3.0.dist-info/METADATA +290 -0
- prefixwise-0.3.0.dist-info/RECORD +17 -0
- prefixwise-0.3.0.dist-info/WHEEL +4 -0
- prefixwise-0.3.0.dist-info/entry_points.txt +3 -0
- prefixwise-0.3.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Native provider diagnostics chaining (v2).
|
|
2
|
+
|
|
3
|
+
The proxy auto-chains the provider's own diagnostic identifier so the
|
|
4
|
+
provider's miss reason is recorded beside Prefixwise's cause on every
|
|
5
|
+
turn. The provider reason is a *partial agreement signal*, not ground
|
|
6
|
+
truth: providers report the earliest divergence only, in a coarse
|
|
7
|
+
taxonomy, and neither names TTL expiry, routing, or eviction.
|
|
8
|
+
|
|
9
|
+
Anthropic (Claude API only, not Bedrock / Google-managed):
|
|
10
|
+
- GA Sep 23, 2026. The beta header ``cache-diagnosis-2026-04-07`` is
|
|
11
|
+
NO LONGER REQUIRED. Send ``diagnostics: {"previous_message_id": id}``
|
|
12
|
+
(first turn: null to opt in). Chaining does not change caching
|
|
13
|
+
behaviour or load prior content. ZDR-eligible.
|
|
14
|
+
- Reasons (6): model_changed, system_changed, tools_changed,
|
|
15
|
+
messages_changed, previous_message_not_found, unavailable.
|
|
16
|
+
``unavailable`` fires when model/system/tools match but tool_choice,
|
|
17
|
+
thinking, context_management, output_config, output_format, or beta
|
|
18
|
+
headers differ: that is PARAM_RENDERED.
|
|
19
|
+
|
|
20
|
+
OpenAI (Responses API only, GPT-5.6+):
|
|
21
|
+
- Send ``prompt_cache_options.comparison_response_id`` with a baseline
|
|
22
|
+
response id. Does not load the conversation or change caching.
|
|
23
|
+
Streaming: read ``prompt_cache_diagnostics`` from ``event.response``
|
|
24
|
+
in the ``response.completed`` event.
|
|
25
|
+
- Types (4): cache_hit, cache_miss, comparison_response_not_found,
|
|
26
|
+
unavailable. Reasons (9): model_changed, prompt_cache_key_changed,
|
|
27
|
+
service_tier_changed, tools_changed, text_format_changed,
|
|
28
|
+
reasoning_effort_changed, verbosity_changed, context_compacted,
|
|
29
|
+
input_changed. No TTL-expiry or routing reason exists.
|
|
30
|
+
|
|
31
|
+
Gemini: no miss diagnostics (verified Nov 2026). Only a hit count.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
from dataclasses import dataclass, field
|
|
37
|
+
|
|
38
|
+
from .classifier import MissCause
|
|
39
|
+
|
|
40
|
+
ANTHROPIC_REASONS = {
|
|
41
|
+
"model_changed": MissCause.MODEL_SWITCH,
|
|
42
|
+
"system_changed": MissCause.SYSTEM_EDIT,
|
|
43
|
+
"tools_changed": MissCause.TOOL_CATALOG_CHANGE, # or TOOL_RESERIALIZE; ambiguous, see below
|
|
44
|
+
"messages_changed": MissCause.HISTORY_EDIT, # or COMPACTION_REWRITE; hook decides
|
|
45
|
+
"previous_message_not_found": MissCause.UNKNOWN,
|
|
46
|
+
"unavailable": MissCause.PARAM_RENDERED,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
OPENAI_REASONS = {
|
|
50
|
+
"model_changed": MissCause.MODEL_SWITCH,
|
|
51
|
+
"prompt_cache_key_changed": MissCause.UNKNOWN,
|
|
52
|
+
"service_tier_changed": MissCause.ROUTING_MISS,
|
|
53
|
+
"tools_changed": MissCause.TOOL_CATALOG_CHANGE,
|
|
54
|
+
"text_format_changed": MissCause.PARAM_RENDERED,
|
|
55
|
+
"reasoning_effort_changed": MissCause.PARAM_RENDERED,
|
|
56
|
+
"verbosity_changed": MissCause.PARAM_RENDERED,
|
|
57
|
+
"context_compacted": MissCause.COMPACTION_REWRITE,
|
|
58
|
+
"input_changed": MissCause.HISTORY_EDIT,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def anthropic_diagnostics(previous_message_id: str | None) -> dict:
|
|
63
|
+
"""Build the Anthropic diagnostics object. No beta header needed (GA Sep 23 2026)."""
|
|
64
|
+
return {"diagnostics": {"previous_message_id": previous_message_id}}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def openai_comparison_options(comparison_response_id: str | None) -> dict:
|
|
68
|
+
"""Build OpenAI prompt_cache_options for diagnostics chaining."""
|
|
69
|
+
if comparison_response_id is None:
|
|
70
|
+
return {}
|
|
71
|
+
return {"prompt_cache_options": {"comparison_response_id": comparison_response_id}}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# Provider reasons are coarse; Prefixwise causes refine them. A Prefixwise
|
|
75
|
+
# cause agrees with a native reason when it is one of the causes that
|
|
76
|
+
# reason covers: Anthropic's system_changed covers both a wording edit and
|
|
77
|
+
# a date-only change; OpenAI's input_changed covers any change to the
|
|
78
|
+
# prompt input (instructions, history, tool output, compaction). Each set
|
|
79
|
+
# lists only causes that are a kind of that reason, never a different
|
|
80
|
+
# block, so agreement still requires naming the right place.
|
|
81
|
+
_CONTENT_CHANGES = {MissCause.HISTORY_EDIT, MissCause.COMPACTION_REWRITE, MissCause.TOOL_RESULT_MUTATION}
|
|
82
|
+
_SYSTEM_CHANGES = {MissCause.SYSTEM_EDIT, MissCause.VOLATILE_SYSTEM}
|
|
83
|
+
_TOOL_CHANGES = {MissCause.TOOL_RESERIALIZE, MissCause.TOOL_CATALOG_CHANGE}
|
|
84
|
+
|
|
85
|
+
NATIVE_COVERS = {
|
|
86
|
+
"anthropic": {
|
|
87
|
+
"model_changed": {MissCause.MODEL_SWITCH},
|
|
88
|
+
"system_changed": _SYSTEM_CHANGES,
|
|
89
|
+
"tools_changed": _TOOL_CHANGES,
|
|
90
|
+
"messages_changed": _CONTENT_CHANGES,
|
|
91
|
+
"previous_message_not_found": {MissCause.UNKNOWN},
|
|
92
|
+
"unavailable": {MissCause.PARAM_RENDERED},
|
|
93
|
+
},
|
|
94
|
+
"openai": {
|
|
95
|
+
"model_changed": {MissCause.MODEL_SWITCH},
|
|
96
|
+
"prompt_cache_key_changed": {MissCause.ROUTING_MISS, MissCause.UNKNOWN},
|
|
97
|
+
"service_tier_changed": {MissCause.ROUTING_MISS, MissCause.PARAM_RENDERED},
|
|
98
|
+
"tools_changed": _TOOL_CHANGES,
|
|
99
|
+
"text_format_changed": {MissCause.PARAM_RENDERED},
|
|
100
|
+
"reasoning_effort_changed": {MissCause.PARAM_RENDERED},
|
|
101
|
+
"verbosity_changed": {MissCause.PARAM_RENDERED},
|
|
102
|
+
"context_compacted": {MissCause.COMPACTION_REWRITE},
|
|
103
|
+
"input_changed": _CONTENT_CHANGES | _SYSTEM_CHANGES,
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def native_covers(provider: str, reason: str | None) -> set[MissCause] | None:
|
|
109
|
+
"""The Prefixwise causes a native reason covers, or None if unmapped."""
|
|
110
|
+
if reason is None:
|
|
111
|
+
return None
|
|
112
|
+
return NATIVE_COVERS.get(provider, {}).get(reason)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def map_native_reason(provider: str, reason: str | None) -> MissCause | None:
|
|
116
|
+
if reason is None:
|
|
117
|
+
return None
|
|
118
|
+
table = {"anthropic": ANTHROPIC_REASONS, "openai": OPENAI_REASONS}.get(provider, {})
|
|
119
|
+
return table.get(reason)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass
|
|
123
|
+
class AgreementScorer:
|
|
124
|
+
"""Score the classifier against native reasons where both exist.
|
|
125
|
+
|
|
126
|
+
Two metrics ship with the classifier:
|
|
127
|
+
- agreement: share of turns where Prefixwise's cause maps onto the
|
|
128
|
+
native reason when both exist (E1 target >= 95%).
|
|
129
|
+
- coverage: share of misses not UNKNOWN (E1 target: UNKNOWN <= 10%).
|
|
130
|
+
A miss providers lump into ``unavailable`` that Prefixwise names
|
|
131
|
+
PARAM_RENDERED counts as agreement when the parameter diff is
|
|
132
|
+
present in the request.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
total_with_native: int = 0
|
|
136
|
+
agreed: int = 0
|
|
137
|
+
total_misses: int = 0
|
|
138
|
+
unknown_misses: int = 0
|
|
139
|
+
|
|
140
|
+
def record(self, cause: MissCause, native_reason: str | None, provider: str) -> bool | None:
|
|
141
|
+
"""Record one turn. Returns True/False for agreement, None if no native reason."""
|
|
142
|
+
is_miss = cause not in (MissCause.NONE, MissCause.FIRST_WRITE)
|
|
143
|
+
if is_miss:
|
|
144
|
+
self.total_misses += 1
|
|
145
|
+
if cause is MissCause.UNKNOWN:
|
|
146
|
+
self.unknown_misses += 1
|
|
147
|
+
if native_reason is None:
|
|
148
|
+
return None
|
|
149
|
+
covered = native_covers(provider, native_reason)
|
|
150
|
+
if covered is None:
|
|
151
|
+
return None
|
|
152
|
+
self.total_with_native += 1
|
|
153
|
+
agreed = cause in covered
|
|
154
|
+
if agreed:
|
|
155
|
+
self.agreed += 1
|
|
156
|
+
return agreed
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def agreement_rate(self) -> float | None:
|
|
160
|
+
if not self.total_with_native:
|
|
161
|
+
return None
|
|
162
|
+
return self.agreed / self.total_with_native
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def unknown_rate(self) -> float | None:
|
|
166
|
+
if not self.total_misses:
|
|
167
|
+
return None
|
|
168
|
+
return self.unknown_misses / self.total_misses
|
prefixwise/explain.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Offline miss explanation: why would (or did) the cache miss between two requests?
|
|
2
|
+
|
|
3
|
+
No proxy, no API key. Give it request bodies as the client sent them,
|
|
4
|
+
optionally with the usage the provider returned, and it runs the same
|
|
5
|
+
fingerprinting and classifier as the proxy.
|
|
6
|
+
|
|
7
|
+
Without usage, the explanation is about the prompt itself: it assumes the
|
|
8
|
+
cache would have served the whole shared prefix, so the answer is "what in
|
|
9
|
+
the request breaks reuse", not "what the provider actually did". TTL
|
|
10
|
+
expiry, eviction, and routing need real usage or engine events.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from .classifier import RequestSnapshot, classify
|
|
20
|
+
from .fingerprint import BlockFingerprint, Fingerprinter
|
|
21
|
+
from .proxy import (cache_ttl_seconds, completion_tokens_of, extract_blocks, has_cache_breakpoint,
|
|
22
|
+
normalize_usage)
|
|
23
|
+
|
|
24
|
+
PARAM_KEYS = ("tool_choice", "thinking", "reasoning", "output_format", "text_format", "verbosity")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def detect_provider(body: dict) -> str:
|
|
28
|
+
"""Anthropic Messages bodies carry system at the top level; OpenAI ones do not."""
|
|
29
|
+
return "anthropic" if "system" in body else "openai"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Turn:
|
|
34
|
+
body: dict
|
|
35
|
+
usage: dict | None = None
|
|
36
|
+
compaction_event: bool = False
|
|
37
|
+
gap_seconds: float | None = None
|
|
38
|
+
session_id: str = "default"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class Explanation:
|
|
43
|
+
cause: str
|
|
44
|
+
legit: bool | None
|
|
45
|
+
evidence: str
|
|
46
|
+
divergence: dict | None
|
|
47
|
+
provider: str
|
|
48
|
+
offline: bool
|
|
49
|
+
prompt_tokens: int | None = None
|
|
50
|
+
cached_tokens: int | None = None
|
|
51
|
+
secondary: list[str] = field(default_factory=list)
|
|
52
|
+
|
|
53
|
+
def to_dict(self) -> dict:
|
|
54
|
+
return dict(self.__dict__)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def first_divergence(prev: list[BlockFingerprint], curr: list[BlockFingerprint]) -> dict | None:
|
|
58
|
+
"""Locate the first block, in prefix order, where two requests differ."""
|
|
59
|
+
for i, (a, b) in enumerate(zip(prev, curr)):
|
|
60
|
+
if a.role != b.role:
|
|
61
|
+
return {"index": i, "role": f"{a.role} -> {b.role}", "kind": "block order changed"}
|
|
62
|
+
if a.raw == b.raw:
|
|
63
|
+
continue
|
|
64
|
+
if a.canonical == b.canonical:
|
|
65
|
+
kind = "same content, different bytes (key order / whitespace / serialization)"
|
|
66
|
+
elif a.masked == b.masked:
|
|
67
|
+
kind = "only volatile values changed (dates, times, ids, numbers)"
|
|
68
|
+
else:
|
|
69
|
+
kind = "content changed"
|
|
70
|
+
return {"index": i, "role": a.role, "kind": kind}
|
|
71
|
+
if len(curr) > len(prev):
|
|
72
|
+
return {"index": len(prev), "role": curr[len(prev)].role,
|
|
73
|
+
"kind": "appended after the previous request (normal growth, not a divergence)"}
|
|
74
|
+
if len(curr) < len(prev):
|
|
75
|
+
return {"index": len(curr), "role": prev[len(curr)].role,
|
|
76
|
+
"kind": "previous blocks removed (history truncated or compacted)"}
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _snapshot(turn: Turn, provider: str, fp: Fingerprinter) -> RequestSnapshot:
|
|
81
|
+
blocks = fp.fingerprint_blocks(extract_blocks(turn.body, provider))
|
|
82
|
+
if turn.usage is not None:
|
|
83
|
+
prompt, cached = normalize_usage(provider, turn.usage)
|
|
84
|
+
min_prefix = None
|
|
85
|
+
else:
|
|
86
|
+
# Offline: pretend the shared prefix was fully served, so only a
|
|
87
|
+
# divergence in the request itself can explain a miss.
|
|
88
|
+
prompt, cached, min_prefix = 1, 1, 0
|
|
89
|
+
return RequestSnapshot(model=str(turn.body.get("model", "unknown")), provider=provider,
|
|
90
|
+
blocks=blocks, prompt_tokens=prompt, cached_tokens=cached,
|
|
91
|
+
gap_seconds=turn.gap_seconds, compaction_event=turn.compaction_event,
|
|
92
|
+
params={k: turn.body.get(k) for k in PARAM_KEYS if turn.body.get(k) is not None},
|
|
93
|
+
has_breakpoint=has_cache_breakpoint(turn.body),
|
|
94
|
+
ttl_seconds=cache_ttl_seconds(provider, turn.body),
|
|
95
|
+
min_prefix_tokens=min_prefix)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def explain_pair(prev: Turn | None, curr: Turn, provider: str | None = None,
|
|
99
|
+
fingerprinter: Fingerprinter | None = None) -> Explanation:
|
|
100
|
+
provider = provider or detect_provider(curr.body)
|
|
101
|
+
fp = fingerprinter or Fingerprinter()
|
|
102
|
+
curr_snap = _snapshot(curr, provider, fp)
|
|
103
|
+
prev_snap = _snapshot(prev, provider, fp) if prev is not None else None
|
|
104
|
+
result = classify(prev_snap, curr_snap)
|
|
105
|
+
div = first_divergence(prev_snap.blocks, curr_snap.blocks) if prev_snap else None
|
|
106
|
+
if div and result.cause.value == "tool_reserialize" and div["kind"] == "content changed":
|
|
107
|
+
div["kind"] = "tools reordered (same tool set, different order)"
|
|
108
|
+
return Explanation(cause=result.cause.value, legit=result.legit, evidence=result.evidence,
|
|
109
|
+
divergence=div, provider=provider, offline=curr.usage is None,
|
|
110
|
+
prompt_tokens=curr_snap.prompt_tokens if curr.usage is not None else None,
|
|
111
|
+
cached_tokens=curr_snap.cached_tokens if curr.usage is not None else None,
|
|
112
|
+
secondary=[c.value for c in result.secondary])
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _turn_from_record(rec: dict) -> Turn:
|
|
116
|
+
"""A record is a bare request body, or {"request"|"body": ..., "usage": ..., ...}."""
|
|
117
|
+
if not isinstance(rec, dict):
|
|
118
|
+
raise ValueError("each record must be a JSON object")
|
|
119
|
+
body = rec.get("request") or rec.get("body")
|
|
120
|
+
if body is None:
|
|
121
|
+
return Turn(body=rec, session_id=str(rec.get("session_id") or "default"))
|
|
122
|
+
usage = rec.get("usage") or (rec.get("response") or {}).get("usage")
|
|
123
|
+
return Turn(body=body, usage=usage,
|
|
124
|
+
compaction_event=bool(rec.get("compaction_event", False)),
|
|
125
|
+
gap_seconds=rec.get("gap_seconds"),
|
|
126
|
+
session_id=str(rec.get("session_id") or body.get("session_id") or "default"))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def load_turns(path: str | Path) -> list[Turn]:
|
|
130
|
+
"""Load one JSON record, a JSON array of records, or JSONL (one record per line)."""
|
|
131
|
+
text = Path(path).read_text()
|
|
132
|
+
stripped = text.strip()
|
|
133
|
+
if not stripped:
|
|
134
|
+
return []
|
|
135
|
+
try:
|
|
136
|
+
data = json.loads(stripped)
|
|
137
|
+
except ValueError:
|
|
138
|
+
data = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
139
|
+
records = data if isinstance(data, list) else [data]
|
|
140
|
+
return [_turn_from_record(r) for r in records]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def explain_session(turns: list[Turn], provider: str | None = None) -> list[Explanation]:
|
|
144
|
+
"""Explain every turn against the previous turn in the same session."""
|
|
145
|
+
fp = Fingerprinter()
|
|
146
|
+
last: dict[str, Turn] = {}
|
|
147
|
+
out = []
|
|
148
|
+
for t in turns:
|
|
149
|
+
out.append(explain_pair(last.get(t.session_id), t, provider, fp))
|
|
150
|
+
last[t.session_id] = t
|
|
151
|
+
return out
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
__all__ = ["Turn", "Explanation", "explain_pair", "explain_session", "load_turns",
|
|
155
|
+
"first_divergence", "detect_provider", "completion_tokens_of"]
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""Canonical block fingerprinting (v2).
|
|
2
|
+
|
|
3
|
+
Split a rendered request into blocks in prefix order and store three
|
|
4
|
+
hashes per block:
|
|
5
|
+
|
|
6
|
+
- ``raw``: SHA-256 (or HMAC) over the exact bytes as sent. This is the
|
|
7
|
+
only hash that can see TOOL_RESERIALIZE, which is defined as
|
|
8
|
+
canonical-equal, raw-different.
|
|
9
|
+
- ``canonical``: SHA-256 (or HMAC) over canonicalised bytes (sorted keys,
|
|
10
|
+
fixed separators, NFC unicode, no trailing-whitespace variance).
|
|
11
|
+
- ``masked``: hash over canonical bytes with volatile values masked out
|
|
12
|
+
(dates, times, numbers, UUIDs, ids). A system block whose raw hash
|
|
13
|
+
changed but whose masked hash is equal is VOLATILE_SYSTEM, not a
|
|
14
|
+
SYSTEM_EDIT.
|
|
15
|
+
|
|
16
|
+
Hashes are stored, never raw prompts, by default.
|
|
17
|
+
|
|
18
|
+
Privacy: a bare hash of a short or predictable block (a four-word user
|
|
19
|
+
message, a boilerplate system line) is dictionary-guessable. When an
|
|
20
|
+
HMAC key is supplied, every hash of a short/predictable block is an
|
|
21
|
+
HMAC with the per-install key, replacing bare SHA-256 entirely for
|
|
22
|
+
that block (raw, canonical, and masked). The short-block threshold
|
|
23
|
+
is a policy choice (default 64 tokens, approximate, ~256 characters),
|
|
24
|
+
not a verified fact; a 60-token boilerplate line can still be
|
|
25
|
+
predictable, and a long block can still be guessable.
|
|
26
|
+
|
|
27
|
+
Soft tokens / summaries are fingerprinted as distinct block types:
|
|
28
|
+
their content is never hashed as text. A soft-token block hashes
|
|
29
|
+
only its type and token count.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import hashlib
|
|
35
|
+
import hmac as hmac_mod
|
|
36
|
+
import json
|
|
37
|
+
import re
|
|
38
|
+
import unicodedata
|
|
39
|
+
from dataclasses import dataclass, field
|
|
40
|
+
|
|
41
|
+
BLOCK_ROLES = (
|
|
42
|
+
"system",
|
|
43
|
+
"system_dynamic",
|
|
44
|
+
"tools",
|
|
45
|
+
"messages",
|
|
46
|
+
"tool_results",
|
|
47
|
+
"user",
|
|
48
|
+
"summary",
|
|
49
|
+
"soft_tokens",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# Policy choice, not a verified fact (see module docstring).
|
|
53
|
+
DEFAULT_SHORT_BLOCK_CHARS = 256
|
|
54
|
+
|
|
55
|
+
_DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?")
|
|
56
|
+
_TIME_RE = re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AP]M)?\b", re.IGNORECASE)
|
|
57
|
+
_UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
|
|
58
|
+
_HEX_ID_RE = re.compile(r"\b[0-9a-fA-F]{16,}\b")
|
|
59
|
+
_NUMBER_RE = re.compile(r"\b\d+(?:\.\d+)?\b")
|
|
60
|
+
_TOKEN_WITH_DIGIT_RE = re.compile(r"\b\w*\d\w*\b")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def mask_volatiles(text: str) -> str:
|
|
64
|
+
"""Replace volatile values (dates, times, ids, numbers) with placeholders."""
|
|
65
|
+
text = _DATE_RE.sub("<DATE>", text)
|
|
66
|
+
text = _TIME_RE.sub("<TIME>", text)
|
|
67
|
+
text = _UUID_RE.sub("<ID>", text)
|
|
68
|
+
text = _HEX_ID_RE.sub("<ID>", text)
|
|
69
|
+
text = _NUMBER_RE.sub("<N>", text)
|
|
70
|
+
# Tokens like abc123 / session-42 are ids, not stable words.
|
|
71
|
+
text = _TOKEN_WITH_DIGIT_RE.sub("<ID>", text)
|
|
72
|
+
return text
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def canonical_bytes(value: object) -> bytes:
|
|
76
|
+
text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
77
|
+
text = unicodedata.normalize("NFC", text)
|
|
78
|
+
return text.encode("utf-8")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def raw_bytes(value: object) -> bytes:
|
|
82
|
+
"""Exact bytes as sent: JSON without key sorting, as the caller rendered it."""
|
|
83
|
+
if isinstance(value, (str, bytes)):
|
|
84
|
+
raw = value if isinstance(value, bytes) else value.encode("utf-8")
|
|
85
|
+
else:
|
|
86
|
+
raw = json.dumps(value, ensure_ascii=False).encode("utf-8")
|
|
87
|
+
return raw
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _hash(data: bytes, hmac_key: bytes | None) -> str:
|
|
91
|
+
if hmac_key:
|
|
92
|
+
return hmac_mod.new(hmac_key, data, hashlib.sha256).hexdigest()
|
|
93
|
+
return hashlib.sha256(data).hexdigest()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class BlockFingerprint:
|
|
98
|
+
role: str
|
|
99
|
+
index: int
|
|
100
|
+
raw: str
|
|
101
|
+
canonical: str
|
|
102
|
+
masked: str
|
|
103
|
+
byte_length: int
|
|
104
|
+
hmac: bool = False
|
|
105
|
+
# First-divergence position in tokens is filled by the caller when a
|
|
106
|
+
# tokenizer is available; None means "not measured", never a guess.
|
|
107
|
+
token_estimate: int | None = None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass
|
|
111
|
+
class Fingerprinter:
|
|
112
|
+
"""Per-install fingerprinter. Supply hmac_key to protect short blocks."""
|
|
113
|
+
|
|
114
|
+
hmac_key: bytes | None = None
|
|
115
|
+
short_block_chars: int = DEFAULT_SHORT_BLOCK_CHARS
|
|
116
|
+
|
|
117
|
+
def _is_short(self, raw: bytes) -> bool:
|
|
118
|
+
return len(raw) <= self.short_block_chars
|
|
119
|
+
|
|
120
|
+
def fingerprint_block(self, role: str, index: int, value: object) -> BlockFingerprint:
|
|
121
|
+
if role == "soft_tokens":
|
|
122
|
+
# Never hash soft-token content as text: hash type + count only.
|
|
123
|
+
count = value if isinstance(value, int) else len(value) # type: ignore[arg-type]
|
|
124
|
+
payload = canonical_bytes({"type": "soft_tokens", "count": count})
|
|
125
|
+
digest = _hash(payload, self.hmac_key)
|
|
126
|
+
return BlockFingerprint(role=role, index=index, raw=digest, canonical=digest,
|
|
127
|
+
masked=digest, byte_length=len(payload), hmac=bool(self.hmac_key))
|
|
128
|
+
raw = raw_bytes(value)
|
|
129
|
+
canon = canonical_bytes(value)
|
|
130
|
+
masked_text = mask_volatiles(canon.decode("utf-8", "replace"))
|
|
131
|
+
masked = masked_text.encode("utf-8")
|
|
132
|
+
# HMAC replaces bare hashing for short/predictable blocks entirely.
|
|
133
|
+
key = self.hmac_key if self._is_short(raw) else None
|
|
134
|
+
return BlockFingerprint(
|
|
135
|
+
role=role, index=index,
|
|
136
|
+
raw=_hash(raw, key), canonical=_hash(canon, key), masked=_hash(masked, key),
|
|
137
|
+
byte_length=len(raw), hmac=bool(key),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
def fingerprint_blocks(self, blocks: list[tuple[str, object]]) -> list[BlockFingerprint]:
|
|
141
|
+
return [self.fingerprint_block(role, i, value) for i, (role, value) in enumerate(blocks)]
|
|
142
|
+
|
|
143
|
+
def prefix_hash(self, fingerprints: list[BlockFingerprint]) -> str:
|
|
144
|
+
joined = "".join(f.canonical for f in fingerprints).encode("utf-8")
|
|
145
|
+
return _hash(joined, self.hmac_key)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def block_hash(value: object) -> str:
|
|
149
|
+
"""Backwards-compatible helper: canonical SHA-256 of a value."""
|
|
150
|
+
return hashlib.sha256(canonical_bytes(value)).hexdigest()
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def fingerprint_blocks(blocks: list[tuple[str, object]], hmac_key: bytes | None = None) -> list[BlockFingerprint]:
|
|
154
|
+
return Fingerprinter(hmac_key=hmac_key).fingerprint_blocks(blocks)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def prefix_hash(fingerprints: list[BlockFingerprint]) -> str:
|
|
158
|
+
joined = "".join(f.canonical for f in fingerprints).encode("utf-8")
|
|
159
|
+
return hashlib.sha256(joined).hexdigest()
|
prefixwise/fixes.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Fix toggles and fix advice.
|
|
2
|
+
|
|
3
|
+
A toggle is a rewrite the proxy can apply to a request before forwarding
|
|
4
|
+
it, opt-in and reversible (turn it off and requests go through unchanged).
|
|
5
|
+
Only rewrites that cannot change what the prompt means are toggles:
|
|
6
|
+
|
|
7
|
+
- canonical_tools: tools sorted by name, each tool's keys sorted. The
|
|
8
|
+
provider caches by exact prefix, and tools come first in every
|
|
9
|
+
provider's prefix, so a tool list whose order or key order drifts
|
|
10
|
+
between turns (plugin load order, dict/set iteration, a framework
|
|
11
|
+
rebuilding definitions) misses the whole cache. Sorting is stable:
|
|
12
|
+
same tool set, same bytes, every turn.
|
|
13
|
+
|
|
14
|
+
Everything else is advice: changes only the application can make safely
|
|
15
|
+
(moving a timestamp out of the system prompt changes the text the model
|
|
16
|
+
reads). FIX_ADVICE maps each wasteful cause to its fix, and says whether
|
|
17
|
+
the proxy can apply it.
|
|
18
|
+
|
|
19
|
+
A fix ships as "recommended" on predicted savings (the avoidable tokens
|
|
20
|
+
the ledger attributes to its cause) and is "proven" only after a replay
|
|
21
|
+
(experiments/e4) shows realised savings of at least half the prediction
|
|
22
|
+
with task success flat.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
|
|
29
|
+
AVAILABLE_FIXES = ("canonical_tools",)
|
|
30
|
+
|
|
31
|
+
# Evidence per fix. Change only with a committed E4 run behind it.
|
|
32
|
+
FIX_STATUS = {
|
|
33
|
+
"canonical_tools": {
|
|
34
|
+
"status": "proven",
|
|
35
|
+
"evidence": "E4 live 2026-11-06: 99.9% of predicted savings realised on Anthropic (Haiku 5.5), "
|
|
36
|
+
"97.5% on vLLM (Qwen3-4B); task success flat (0/60 and 1/60 turns changed). "
|
|
37
|
+
"experiments/e4/runs/2026-11-06-live.md",
|
|
38
|
+
},
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _tool_name(tool: object) -> str | None:
|
|
43
|
+
if not isinstance(tool, dict):
|
|
44
|
+
return None
|
|
45
|
+
fn = tool.get("function")
|
|
46
|
+
if isinstance(fn, dict) and fn.get("name"):
|
|
47
|
+
return str(fn["name"]) # OpenAI chat completions
|
|
48
|
+
return str(tool["name"]) if tool.get("name") else None # Anthropic, Responses API
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _sorted_keys(value: object) -> object:
|
|
52
|
+
# json round trip with sort_keys: nested dicts come back in sorted key order.
|
|
53
|
+
return json.loads(json.dumps(value, sort_keys=True))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def canonical_tools(tools: list) -> list:
|
|
57
|
+
"""Tools sorted by name (unnamed tools keep their order, at the end),
|
|
58
|
+
with every tool's keys sorted. Same set in, same bytes out."""
|
|
59
|
+
named = sorted((t for t in tools if _tool_name(t) is not None), key=_tool_name)
|
|
60
|
+
unnamed = [t for t in tools if _tool_name(t) is None]
|
|
61
|
+
return [_sorted_keys(t) for t in named] + [_sorted_keys(t) for t in unnamed]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def apply_fixes(body: dict, fixes: tuple[str, ...] | list[str]) -> tuple[dict, list[str]]:
|
|
65
|
+
"""Return (body to forward, fixes that changed it). Never mutates body."""
|
|
66
|
+
applied: list[str] = []
|
|
67
|
+
out = body
|
|
68
|
+
if "canonical_tools" in fixes and isinstance(body.get("tools"), list) and body["tools"]:
|
|
69
|
+
tools = canonical_tools(body["tools"])
|
|
70
|
+
if tools != body["tools"] or json.dumps(tools) != json.dumps(body["tools"]):
|
|
71
|
+
out = {**out, "tools": tools}
|
|
72
|
+
applied.append("canonical_tools")
|
|
73
|
+
return out, applied
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def parse_fixes(spec: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]:
|
|
77
|
+
"""'canonical_tools,other' or a list -> validated tuple. Unknown names raise."""
|
|
78
|
+
if not spec:
|
|
79
|
+
return ()
|
|
80
|
+
names = [s.strip() for s in (spec.split(",") if isinstance(spec, str) else spec) if s and s.strip()]
|
|
81
|
+
unknown = [n for n in names if n not in AVAILABLE_FIXES]
|
|
82
|
+
if unknown:
|
|
83
|
+
raise ValueError(f"unknown fix {unknown}; available: {', '.join(AVAILABLE_FIXES)}")
|
|
84
|
+
return tuple(dict.fromkeys(names))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# Cause -> how to fix it. "toggle" names a proxy fix that applies it for you.
|
|
88
|
+
FIX_ADVICE: dict[str, dict] = {
|
|
89
|
+
"tool_reserialize": {"toggle": "canonical_tools",
|
|
90
|
+
"fix": "Send tools in a fixed order with deterministic serialization "
|
|
91
|
+
"(or let the proxy do it: prefixwise serve --fix canonical_tools)."},
|
|
92
|
+
"volatile_system": {"toggle": None,
|
|
93
|
+
"fix": "Move dates, times and ids out of the system prompt, to the end of the "
|
|
94
|
+
"prompt or the latest user message."},
|
|
95
|
+
"tool_result_mutation": {"toggle": None,
|
|
96
|
+
"fix": "Store each tool result once, exactly as first sent; never re-render it."},
|
|
97
|
+
"history_edit": {"toggle": None,
|
|
98
|
+
"fix": "Append to the conversation; do not rewrite earlier messages. If you must, "
|
|
99
|
+
"send X-Prefixwise-Compaction-Event: 1 and compact at task boundaries."},
|
|
100
|
+
"retry_replay": {"toggle": None,
|
|
101
|
+
"fix": "Check retry logic: identical requests within seconds are re-billed in full."},
|
|
102
|
+
"breakpoint_misplaced": {"toggle": None,
|
|
103
|
+
"fix": "Anthropic: add a cache_control breakpoint after the stable prefix "
|
|
104
|
+
"(tools and system)."},
|
|
105
|
+
}
|