tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
tokencut/gateway/note.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""A fixed system note that tells the model how to treat TokenCut's markers (prompt enhancement).
|
|
2
|
+
|
|
3
|
+
Measured harms it targets: after an `omitted` marker the model re-reads the whole file, or
|
|
4
|
+
re-runs a command to see cut log lines (live run L3). The note is the same text on every
|
|
5
|
+
request and sits at the end of the system prompt, never in a user message, so it is cached
|
|
6
|
+
with the rest of the prefix (~0.1x price after the first request). Only agent requests (two or
|
|
7
|
+
more client tools) get it; side requests (titles, summaries) are left alone.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from tokencut.gateway.history import _tool_lists
|
|
11
|
+
from tokencut.gateway.virtual import MIN_CLIENT_TOOLS, _is_server_tool
|
|
12
|
+
from tokencut.tokens import count_tokens
|
|
13
|
+
|
|
14
|
+
SYSTEM_NOTE = (
|
|
15
|
+
"TokenCut note: tool output in this conversation may be shortened to save tokens. A line "
|
|
16
|
+
"like [TokenCut: lines A-B of path omitted ...] means those lines exist but were not sent; "
|
|
17
|
+
"if you need them, read only lines A-B (for example Read with offset/limit, or "
|
|
18
|
+
"sed -n 'A,Bp' path), not the whole file again. Shortened command output keeps the errors, "
|
|
19
|
+
"failures and final summary: do not re-run a command only to see the omitted lines. Prefer "
|
|
20
|
+
"reading files in line ranges, and keep test output short (for example pytest -q --tb=short)."
|
|
21
|
+
)
|
|
22
|
+
_SYSTEM_ROLES = ("system", "developer")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def add_system_note(provider: str, body: dict) -> dict | None:
|
|
26
|
+
"""Add SYSTEM_NOTE to an agent request's system prompt, in place. Report entry or None."""
|
|
27
|
+
if not _is_agent_request(body):
|
|
28
|
+
return None
|
|
29
|
+
if provider == "anthropic":
|
|
30
|
+
system = body.get("system")
|
|
31
|
+
if isinstance(system, list):
|
|
32
|
+
if any(isinstance(b, dict) and b.get("text") == SYSTEM_NOTE for b in system):
|
|
33
|
+
return None
|
|
34
|
+
system.append({"type": "text", "text": SYSTEM_NOTE})
|
|
35
|
+
elif isinstance(system, str) and system:
|
|
36
|
+
if SYSTEM_NOTE in system:
|
|
37
|
+
return None
|
|
38
|
+
body["system"] = f"{system}\n\n{SYSTEM_NOTE}"
|
|
39
|
+
elif not system:
|
|
40
|
+
body["system"] = SYSTEM_NOTE
|
|
41
|
+
else:
|
|
42
|
+
return None
|
|
43
|
+
elif provider == "openai":
|
|
44
|
+
messages = body.get("messages")
|
|
45
|
+
if not isinstance(messages, list):
|
|
46
|
+
return None # not Chat Completions
|
|
47
|
+
if any(isinstance(m, dict) and m.get("content") == SYSTEM_NOTE for m in messages):
|
|
48
|
+
return None
|
|
49
|
+
lead = 0
|
|
50
|
+
while lead < len(messages) and isinstance(messages[lead], dict) \
|
|
51
|
+
and messages[lead].get("role") in _SYSTEM_ROLES:
|
|
52
|
+
lead += 1
|
|
53
|
+
role = messages[0]["role"] if lead else "system" # "developer" where the agent uses it
|
|
54
|
+
messages.insert(lead, {"role": role, "content": SYSTEM_NOTE})
|
|
55
|
+
elif provider == "gemini":
|
|
56
|
+
key = "system_instruction" if "system_instruction" in body else "systemInstruction"
|
|
57
|
+
system = body.setdefault(key, {})
|
|
58
|
+
parts = system.setdefault("parts", []) if isinstance(system, dict) else None
|
|
59
|
+
if not isinstance(parts, list) or {"text": SYSTEM_NOTE} in parts:
|
|
60
|
+
return None
|
|
61
|
+
parts.append({"text": SYSTEM_NOTE})
|
|
62
|
+
else:
|
|
63
|
+
return None
|
|
64
|
+
return {"segment": "system", "how": "system_note", "tool": "", "before": 0,
|
|
65
|
+
"after": count_tokens(SYSTEM_NOTE)}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _is_agent_request(body: dict) -> bool:
|
|
69
|
+
client_tools = sum(1 for tools, _ in _tool_lists(body) for t in tools
|
|
70
|
+
if isinstance(t, dict) and not _is_server_tool(t))
|
|
71
|
+
return client_tools >= MIN_CLIENT_TOOLS
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Classify a user's request into a task type (prompt enhancement, docs/PROMPT_ROUTER_LABELS.md).
|
|
2
|
+
|
|
3
|
+
Cascade: cheap rules first (follow-ups like "yes", empty text), then a classifier on frozen
|
|
4
|
+
sentence embeddings (local, fastembed / ONNX on CPU). A prediction that is not confident
|
|
5
|
+
enough goes to `ambiguous`, and the user's message is then left unchanged: a wrong label
|
|
6
|
+
would give the agent wrong advice. Everything here is deterministic for the same text and
|
|
7
|
+
the same saved model, so a message gets the same label on every turn (prompt cache safe).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
AMBIGUOUS = "ambiguous"
|
|
17
|
+
STAGE1 = {
|
|
18
|
+
"bugfix": "change_code",
|
|
19
|
+
"feature": "change_code",
|
|
20
|
+
"refactor": "change_code",
|
|
21
|
+
"tests": "change_code",
|
|
22
|
+
"explain": "understand_code",
|
|
23
|
+
"review": "understand_code",
|
|
24
|
+
"design": "plan",
|
|
25
|
+
"run_action": "run_action",
|
|
26
|
+
"write_text": "write_text",
|
|
27
|
+
"followup": "followup",
|
|
28
|
+
AMBIGUOUS: AMBIGUOUS,
|
|
29
|
+
}
|
|
30
|
+
LABELS = tuple(STAGE1)
|
|
31
|
+
MODEL = "BAAI/bge-small-en-v1.5" # same model as the offline benchmark and the repo index
|
|
32
|
+
EMBED_BATCH = 8
|
|
33
|
+
MAX_CHARS = 2_000 # the start of a request says what is wanted; long pastes follow it
|
|
34
|
+
|
|
35
|
+
_FOLLOWUP_WORDS = frozenset(
|
|
36
|
+
["y", "yes", "yep", "yeah", "yup", "ok", "okay", "k", "sure", "no", "nope", "continue", "go", "ahead", "on", "proceed", "do", "it", "that", "this", "looks", "look", "good", "great", "fine", "lgtm", "thanks", "thank", "you", "please", "option", "use", "the", "first", "second", "third", "one", "right", "correct", "agreed", "next", "sounds", "perfect"]
|
|
37
|
+
)
|
|
38
|
+
_WORD_RE = re.compile(r"[a-z]+|\d+")
|
|
39
|
+
MAX_FOLLOWUP_WORDS = 6
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def rule_label(text: str) -> str | None:
|
|
43
|
+
"""Label decided without a model, or None (then the classifier decides)."""
|
|
44
|
+
words = _WORD_RE.findall(text.lower())
|
|
45
|
+
if not words:
|
|
46
|
+
return AMBIGUOUS
|
|
47
|
+
if len(words) <= MAX_FOLLOWUP_WORDS and all(w in _FOLLOWUP_WORDS or w.isdigit() for w in words):
|
|
48
|
+
return "followup"
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _unit(x: np.ndarray) -> np.ndarray:
|
|
53
|
+
x = np.asarray(x, dtype=np.float32)
|
|
54
|
+
return x / np.maximum(np.linalg.norm(x, axis=1, keepdims=True), 1e-12)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _softmax(z: np.ndarray) -> np.ndarray:
|
|
58
|
+
z = z - z.max(axis=1, keepdims=True)
|
|
59
|
+
e = np.exp(z)
|
|
60
|
+
return e / e.sum(axis=1, keepdims=True)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Prototypes:
|
|
64
|
+
"""One mean vector per label; probability = softmax of scaled cosine similarity."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, temperature: float = 20.0):
|
|
67
|
+
self.temperature = temperature
|
|
68
|
+
|
|
69
|
+
def fit(self, x: np.ndarray, y: list[str]) -> "Prototypes":
|
|
70
|
+
x = _unit(x)
|
|
71
|
+
self.labels = sorted(set(y))
|
|
72
|
+
self.centroids = _unit(np.stack([x[[t == lab for t in y]].mean(axis=0) for lab in self.labels]))
|
|
73
|
+
return self
|
|
74
|
+
|
|
75
|
+
def predict_proba(self, x: np.ndarray) -> np.ndarray:
|
|
76
|
+
return _softmax(self.temperature * _unit(x) @ self.centroids.T)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class KNN:
|
|
80
|
+
"""Similarity-weighted vote of the k nearest training prompts."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, k: int = 5, temperature: float = 20.0):
|
|
83
|
+
self.k, self.temperature = k, temperature
|
|
84
|
+
|
|
85
|
+
def fit(self, x: np.ndarray, y: list[str]) -> "KNN":
|
|
86
|
+
self.x = _unit(x)
|
|
87
|
+
self.labels = sorted(set(y))
|
|
88
|
+
self.y = np.array([self.labels.index(t) for t in y])
|
|
89
|
+
return self
|
|
90
|
+
|
|
91
|
+
def predict_proba(self, x: np.ndarray) -> np.ndarray:
|
|
92
|
+
sims = _unit(x) @ self.x.T
|
|
93
|
+
k = min(self.k, sims.shape[1])
|
|
94
|
+
top = np.argsort(-sims, axis=1, kind="stable")[:, :k]
|
|
95
|
+
out = np.full((len(x), len(self.labels)), 1e-6, dtype=np.float64)
|
|
96
|
+
for i, row in enumerate(top):
|
|
97
|
+
weights = np.exp(self.temperature * (sims[i, row] - sims[i, row].max()))
|
|
98
|
+
np.add.at(out[i], self.y[row], weights)
|
|
99
|
+
return out / out.sum(axis=1, keepdims=True)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class SoftmaxHead:
|
|
103
|
+
"""Multinomial logistic regression on frozen embeddings (full-batch Adam, L2).
|
|
104
|
+
|
|
105
|
+
class_weight="balanced": each label counts as much as any other in the loss, however many
|
|
106
|
+
examples it has (weight = n / (labels * count)), so rare labels are not drowned out."""
|
|
107
|
+
|
|
108
|
+
def __init__(self, l2: float = 1e-3, epochs: int = 500, lr: float = 0.05,
|
|
109
|
+
class_weight: str | None = None):
|
|
110
|
+
self.l2, self.epochs, self.lr, self.class_weight = l2, epochs, lr, class_weight
|
|
111
|
+
|
|
112
|
+
def fit(self, x: np.ndarray, y: list[str]) -> "SoftmaxHead":
|
|
113
|
+
x = _unit(x).astype(np.float64)
|
|
114
|
+
self.labels = sorted(set(y))
|
|
115
|
+
target = np.zeros((len(y), len(self.labels)))
|
|
116
|
+
target[np.arange(len(y)), [self.labels.index(t) for t in y]] = 1
|
|
117
|
+
weight = np.ones((len(y), 1))
|
|
118
|
+
if self.class_weight == "balanced":
|
|
119
|
+
counts = target.sum(axis=0)
|
|
120
|
+
weight = (len(y) / (len(self.labels) * counts))[[self.labels.index(t) for t in y]][:, None]
|
|
121
|
+
w = np.zeros((x.shape[1], len(self.labels)))
|
|
122
|
+
b = np.zeros(len(self.labels))
|
|
123
|
+
m = [np.zeros_like(w), np.zeros_like(b)]
|
|
124
|
+
v = [np.zeros_like(w), np.zeros_like(b)]
|
|
125
|
+
for step in range(1, self.epochs + 1):
|
|
126
|
+
grad = weight * (_softmax(x @ w + b) - target) / len(y)
|
|
127
|
+
grads = [x.T @ grad + self.l2 * w, grad.sum(axis=0)]
|
|
128
|
+
for j, (param, g) in enumerate(zip((w, b), grads)):
|
|
129
|
+
m[j] = 0.9 * m[j] + 0.1 * g
|
|
130
|
+
v[j] = 0.999 * v[j] + 0.001 * g * g
|
|
131
|
+
param -= self.lr * (m[j] / (1 - 0.9**step)) / (np.sqrt(v[j] / (1 - 0.999**step)) + 1e-8)
|
|
132
|
+
self.w, self.b = w.astype(np.float32), b.astype(np.float32)
|
|
133
|
+
return self
|
|
134
|
+
|
|
135
|
+
def predict_proba(self, x: np.ndarray) -> np.ndarray:
|
|
136
|
+
return _softmax(_unit(x) @ self.w + self.b)
|
|
137
|
+
|
|
138
|
+
def save(self, path: Path) -> None:
|
|
139
|
+
np.savez(path, w=self.w, b=self.b, labels=np.array(self.labels))
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def load(cls, path: Path) -> "SoftmaxHead":
|
|
143
|
+
data = np.load(path)
|
|
144
|
+
head = cls()
|
|
145
|
+
head.w, head.b, head.labels = data["w"], data["b"], [str(s) for s in data["labels"]]
|
|
146
|
+
return head
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class TwoStage:
|
|
150
|
+
"""Broad group first (STAGE1), then the label inside the group: p = p(group) * p(label|group)."""
|
|
151
|
+
|
|
152
|
+
def __init__(self, make: Callable[[], object] = SoftmaxHead):
|
|
153
|
+
self.make = make
|
|
154
|
+
|
|
155
|
+
def fit(self, x: np.ndarray, y: list[str]) -> "TwoStage":
|
|
156
|
+
self.labels = sorted(set(y))
|
|
157
|
+
groups = [STAGE1[t] for t in y]
|
|
158
|
+
self.stage1 = self.make().fit(x, groups)
|
|
159
|
+
self.stage2 = {}
|
|
160
|
+
for group in self.stage1.labels:
|
|
161
|
+
rows = [i for i, g in enumerate(groups) if g == group]
|
|
162
|
+
if len({y[i] for i in rows}) > 1:
|
|
163
|
+
self.stage2[group] = self.make().fit(x[rows], [y[i] for i in rows])
|
|
164
|
+
return self
|
|
165
|
+
|
|
166
|
+
def predict_proba(self, x: np.ndarray) -> np.ndarray:
|
|
167
|
+
p1 = self.stage1.predict_proba(x)
|
|
168
|
+
out = np.zeros((len(x), len(self.labels)))
|
|
169
|
+
for g, group in enumerate(self.stage1.labels):
|
|
170
|
+
if group in self.stage2:
|
|
171
|
+
inner = self.stage2[group]
|
|
172
|
+
p2 = inner.predict_proba(x)
|
|
173
|
+
for j, label in enumerate(inner.labels):
|
|
174
|
+
out[:, self.labels.index(label)] = p1[:, g] * p2[:, j]
|
|
175
|
+
else:
|
|
176
|
+
label = next(t for t in self.labels if STAGE1[t] == group)
|
|
177
|
+
out[:, self.labels.index(label)] = p1[:, g]
|
|
178
|
+
return out
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def fastembed_embedder(cache_dir: Path, model: str = MODEL) -> Callable[[list[str]], np.ndarray]:
|
|
182
|
+
from fastembed import TextEmbedding # optional dependency, imported only when used
|
|
183
|
+
|
|
184
|
+
encoder = TextEmbedding(model, cache_dir=str(cache_dir / "models"))
|
|
185
|
+
# Small batches: a long-context model (8k tokens) embedding 256 long prompts at once
|
|
186
|
+
# runs a 16 GB laptop out of memory.
|
|
187
|
+
return lambda texts: np.stack([np.asarray(v, dtype=np.float32)
|
|
188
|
+
for v in encoder.embed(texts, batch_size=EMBED_BATCH)])
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class PromptRouter:
|
|
192
|
+
"""rules -> classifier -> `ambiguous` when the top label is not clearly ahead."""
|
|
193
|
+
|
|
194
|
+
def __init__(self, model, embed: Callable[[list[str]], np.ndarray],
|
|
195
|
+
min_prob: float = 0.6, min_margin: float = 0.15):
|
|
196
|
+
self.model, self.embed = model, embed
|
|
197
|
+
self.min_prob, self.min_margin = min_prob, min_margin
|
|
198
|
+
self._cache: dict[str, tuple[str, float]] = {}
|
|
199
|
+
|
|
200
|
+
def classify(self, text: str) -> tuple[str, float]:
|
|
201
|
+
ruled = rule_label(text)
|
|
202
|
+
if ruled is not None:
|
|
203
|
+
return ruled, 1.0
|
|
204
|
+
key = text[:MAX_CHARS]
|
|
205
|
+
if key not in self._cache:
|
|
206
|
+
probs = self.model.predict_proba(self.embed([key]))[0]
|
|
207
|
+
order = np.argsort(-probs, kind="stable")
|
|
208
|
+
top = float(probs[order[0]])
|
|
209
|
+
second = float(probs[order[1]]) if len(order) > 1 else 0.0
|
|
210
|
+
label = self.model.labels[order[0]]
|
|
211
|
+
if top < self.min_prob or top - second < self.min_margin:
|
|
212
|
+
label = AMBIGUOUS
|
|
213
|
+
self._cache[key] = (label, top)
|
|
214
|
+
return self._cache[key]
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Gateway measurements: what each request contains, and what the provider billed.
|
|
2
|
+
|
|
3
|
+
Only numbers are logged (token counts per segment kind, usage), never prompt text or
|
|
4
|
+
headers, so the log is safe to keep. Full request bodies are saved only when capture is
|
|
5
|
+
switched on, for building parser test fixtures.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import copy
|
|
9
|
+
import dataclasses
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import time
|
|
13
|
+
from collections import deque
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from tokencut.contracts import Segment, SegmentKind, ToolResultKind
|
|
18
|
+
from tokencut.gateway.adapters import anthropic, gemini, openai
|
|
19
|
+
from tokencut.gateway.adapters.common import lookup
|
|
20
|
+
from tokencut.gateway.brief import add_task_briefs
|
|
21
|
+
from tokencut.gateway.history import CacheEventConfig, trim_with_cache_events
|
|
22
|
+
from tokencut.gateway.media import adjust_images, split_pdfs, with_media_tokens
|
|
23
|
+
from tokencut.gateway.note import add_system_note
|
|
24
|
+
from tokencut.gateway.session import SessionStore
|
|
25
|
+
from tokencut.gateway.trimmer import TrimConfig, trim_request
|
|
26
|
+
from tokencut.gateway.virtual import VirtualConfig, finish_tools, offers_tools, resolve
|
|
27
|
+
|
|
28
|
+
log = logging.getLogger(__name__)
|
|
29
|
+
ADAPTERS = {"anthropic": anthropic, "openai": openai, "gemini": gemini}
|
|
30
|
+
RECENT_RECORDS = 1000
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def analyse(
|
|
34
|
+
provider: str,
|
|
35
|
+
path: str,
|
|
36
|
+
body: bytes,
|
|
37
|
+
trim_config: TrimConfig | None = None,
|
|
38
|
+
session: tuple[SessionStore, CacheEventConfig] | None = None,
|
|
39
|
+
now: float | None = None,
|
|
40
|
+
virtual: VirtualConfig | None = None,
|
|
41
|
+
router=None,
|
|
42
|
+
) -> tuple[dict, bytes | None]:
|
|
43
|
+
"""Split a request into segments and count tokens per kind; with a TrimConfig, also
|
|
44
|
+
work out the cache-safe trim and the gateway's own tools (virtual.py). Returns
|
|
45
|
+
(analysis, body bytes to send or None if unchanged).
|
|
46
|
+
Raises on bad input (the caller then forwards the original body)."""
|
|
47
|
+
start = time.perf_counter()
|
|
48
|
+
data = json.loads(body)
|
|
49
|
+
adapter = ADAPTERS[provider]
|
|
50
|
+
answered = [] # the gateway's answers to its own tools (replace the agent's errors)
|
|
51
|
+
if trim_config is not None and virtual is not None:
|
|
52
|
+
answered = resolve(provider, data, virtual.store)
|
|
53
|
+
model = adapter.model_of(data, path)
|
|
54
|
+
# attachments are forwarded unchanged; their estimated provider tokens are counted
|
|
55
|
+
segments, attachments = with_media_tokens(provider, model, data, adapter.parse(data))
|
|
56
|
+
analysis = {
|
|
57
|
+
"model": model,
|
|
58
|
+
"stream": bool(data.get("stream")) or "streamGenerateContent" in path,
|
|
59
|
+
"segments": len(segments),
|
|
60
|
+
**token_breakdown(segments),
|
|
61
|
+
"media": attachments,
|
|
62
|
+
}
|
|
63
|
+
new_bytes = None
|
|
64
|
+
if trim_config is not None:
|
|
65
|
+
trim_start = time.perf_counter()
|
|
66
|
+
event = None
|
|
67
|
+
ccr = virtual is not None and virtual.ccr and offers_tools(provider, data)
|
|
68
|
+
if ccr:
|
|
69
|
+
trim_config = dataclasses.replace(trim_config, ccr=True)
|
|
70
|
+
if session is not None: # cache-event trimming (remembers decisions per conversation)
|
|
71
|
+
store, events_config = session
|
|
72
|
+
new_body, report, event = trim_with_cache_events(
|
|
73
|
+
provider, analysis["model"], data, segments, trim_config, events_config, store,
|
|
74
|
+
now=now, total_tokens=analysis["tokens_total"],
|
|
75
|
+
)
|
|
76
|
+
else:
|
|
77
|
+
new_body, report = trim_request(data, segments, trim_config)
|
|
78
|
+
# Attachments (steps 5-7). Applied last and in descending path order: an image may
|
|
79
|
+
# gain a note and a PDF may become two blocks, which shifts later list positions.
|
|
80
|
+
done = {d["segment"] for d in report.trimmed} # e.g. a duplicate is already a note
|
|
81
|
+
replacements, entries = adjust_images(provider, model, data, segments,
|
|
82
|
+
trim_config.media_user, trim_config.resize_images,
|
|
83
|
+
trim_config.trim_borders)
|
|
84
|
+
if trim_config.pdf_text_pages:
|
|
85
|
+
pdf_changes, pdf_entries = split_pdfs(provider, data, segments, trim_config.media_user)
|
|
86
|
+
replacements, entries = replacements + pdf_changes, entries + pdf_entries
|
|
87
|
+
pending = [(p, b, e) for (p, b), e in zip(replacements, entries, strict=True)
|
|
88
|
+
if e["segment"] not in done]
|
|
89
|
+
for block_path, blocks, entry in sorted(pending, key=lambda c: tuple(c[0]), reverse=True):
|
|
90
|
+
if new_body is None:
|
|
91
|
+
new_body = copy.deepcopy(data)
|
|
92
|
+
parent = lookup(new_body, block_path[:-1])
|
|
93
|
+
parent[block_path[-1]:block_path[-1] + 1] = blocks
|
|
94
|
+
report.tokens_before += entry["before"]
|
|
95
|
+
report.tokens_after += entry["after"]
|
|
96
|
+
report.trimmed.append(entry)
|
|
97
|
+
if virtual is not None:
|
|
98
|
+
if ccr: # keep every original we shortened (fallback if it leaves the request)
|
|
99
|
+
shortened = {d["segment"] for d in report.trimmed}
|
|
100
|
+
for seg in segments:
|
|
101
|
+
if seg.id in shortened and seg.text: # attachments have no text to keep
|
|
102
|
+
virtual.store.put(seg.text)
|
|
103
|
+
out = new_body if new_body is not None else data
|
|
104
|
+
extra = answered + finish_tools(provider, out, ccr, virtual.lazy_tools,
|
|
105
|
+
virtual.core_tools)
|
|
106
|
+
for entry in extra:
|
|
107
|
+
report.tokens_before += entry["before"]
|
|
108
|
+
report.tokens_after += entry["after"]
|
|
109
|
+
report.trimmed.append(entry)
|
|
110
|
+
if extra:
|
|
111
|
+
new_body = out
|
|
112
|
+
if trim_config.prompt_brief and router is not None: # before the note (it shifts paths)
|
|
113
|
+
out = new_body if new_body is not None else copy.deepcopy(data)
|
|
114
|
+
try:
|
|
115
|
+
entries, analysis["task"] = add_task_briefs(provider, out, segments, router,
|
|
116
|
+
trim_config.brief_threshold)
|
|
117
|
+
except Exception as exc: # fail open: no briefs, the request still goes out
|
|
118
|
+
log.warning("task briefs failed", exc_info=True)
|
|
119
|
+
entries, analysis["task"] = [], {"error": type(exc).__name__}
|
|
120
|
+
for entry in entries:
|
|
121
|
+
report.tokens_after += entry["after"]
|
|
122
|
+
report.trimmed.append(entry)
|
|
123
|
+
if entries:
|
|
124
|
+
new_body = out
|
|
125
|
+
if trim_config.system_note: # last: it may insert a message (OpenAI), shifting paths
|
|
126
|
+
out = new_body if new_body is not None else copy.deepcopy(data)
|
|
127
|
+
entry = add_system_note(provider, out)
|
|
128
|
+
if entry is not None:
|
|
129
|
+
report.tokens_after += entry["after"]
|
|
130
|
+
report.trimmed.append(entry)
|
|
131
|
+
new_body = out
|
|
132
|
+
analysis["trim"] = {
|
|
133
|
+
**report.as_dict(),
|
|
134
|
+
"tier": trim_config.tier.value,
|
|
135
|
+
"cache_event": event,
|
|
136
|
+
"trim_ms": round((time.perf_counter() - trim_start) * 1000, 1),
|
|
137
|
+
}
|
|
138
|
+
if new_body is not None:
|
|
139
|
+
new_bytes = json.dumps(new_body, ensure_ascii=False, separators=(",", ":")).encode()
|
|
140
|
+
analysis["analysis_ms"] = round((time.perf_counter() - start) * 1000, 1)
|
|
141
|
+
return analysis, new_bytes
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def token_breakdown(segments: list[Segment]) -> dict:
|
|
145
|
+
by_kind = {kind.value: 0 for kind in SegmentKind}
|
|
146
|
+
by_result = {kind.value: 0 for kind in ToolResultKind}
|
|
147
|
+
for s in segments:
|
|
148
|
+
by_kind[s.kind.value] += s.token_count
|
|
149
|
+
if s.result_kind is not None:
|
|
150
|
+
by_result[s.result_kind.value] += s.token_count
|
|
151
|
+
return {
|
|
152
|
+
"tokens_total": sum(by_kind.values()),
|
|
153
|
+
"tokens_by_kind": by_kind,
|
|
154
|
+
"tool_result_tokens": by_result,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class RequestLog:
|
|
159
|
+
def __init__(self, log_dir: Path | None, capture: bool = False):
|
|
160
|
+
self.log_dir = log_dir
|
|
161
|
+
self.capture = capture
|
|
162
|
+
self.recent: deque[dict] = deque(maxlen=RECENT_RECORDS)
|
|
163
|
+
|
|
164
|
+
def add(self, record: dict) -> None:
|
|
165
|
+
self.recent.append(record)
|
|
166
|
+
if self.log_dir is None:
|
|
167
|
+
return
|
|
168
|
+
self.log_dir.mkdir(parents=True, exist_ok=True)
|
|
169
|
+
day = record["ts"][:10]
|
|
170
|
+
with (self.log_dir / f"requests-{day}.jsonl").open("a", encoding="utf-8") as f:
|
|
171
|
+
f.write(json.dumps(record) + "\n")
|
|
172
|
+
|
|
173
|
+
def save_capture(self, provider: str, body: bytes) -> None:
|
|
174
|
+
"""Keep a full request body (local only; contains the user's code and prompts)."""
|
|
175
|
+
if not self.capture or self.log_dir is None:
|
|
176
|
+
return
|
|
177
|
+
folder = self.log_dir / "captures"
|
|
178
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
179
|
+
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%f")
|
|
180
|
+
(folder / f"{stamp}_{provider}.json").write_bytes(body)
|
|
181
|
+
|
|
182
|
+
def summary(self) -> dict:
|
|
183
|
+
"""Totals over the recent requests, for GET /tokencut/stats."""
|
|
184
|
+
analysed = [r for r in self.recent if r.get("analysis")]
|
|
185
|
+
by_kind = {kind.value: 0 for kind in SegmentKind}
|
|
186
|
+
by_result = {kind.value: 0 for kind in ToolResultKind}
|
|
187
|
+
usage_total = {
|
|
188
|
+
"input_tokens": 0,
|
|
189
|
+
"cache_read_tokens": 0,
|
|
190
|
+
"cache_write_tokens": 0,
|
|
191
|
+
"output_tokens": 0,
|
|
192
|
+
}
|
|
193
|
+
providers: dict[str, int] = {}
|
|
194
|
+
for r in self.recent:
|
|
195
|
+
providers[r["provider"]] = providers.get(r["provider"], 0) + 1
|
|
196
|
+
for field, value in (r.get("usage") or {}).items():
|
|
197
|
+
usage_total[field] = usage_total.get(field, 0) + value
|
|
198
|
+
for r in analysed:
|
|
199
|
+
for kind, n in r["analysis"]["tokens_by_kind"].items():
|
|
200
|
+
by_kind[kind] += n
|
|
201
|
+
for kind, n in r["analysis"]["tool_result_tokens"].items():
|
|
202
|
+
by_result[kind] += n
|
|
203
|
+
attachments = {"images": 0, "documents": 0, "unknown": 0, "tokens": 0}
|
|
204
|
+
for r in analysed:
|
|
205
|
+
for field, value in (r["analysis"].get("media") or {}).items():
|
|
206
|
+
attachments[field] = attachments.get(field, 0) + value
|
|
207
|
+
total = sum(by_kind.values())
|
|
208
|
+
trims = [r["analysis"]["trim"] for r in analysed if r["analysis"].get("trim")]
|
|
209
|
+
applied = [r for r in analysed if r.get("trim_applied")]
|
|
210
|
+
return {
|
|
211
|
+
"trimming": {
|
|
212
|
+
"cache_events": _count(
|
|
213
|
+
[t["cache_event"] for t in trims if t.get("cache_event")]
|
|
214
|
+
),
|
|
215
|
+
"saved_by": _saved_by(trims),
|
|
216
|
+
"requests_with_trims": sum(1 for t in trims if t["segments_trimmed"]),
|
|
217
|
+
"tokens_saved_possible": sum(t["tokens_saved"] for t in trims),
|
|
218
|
+
"tokens_saved_applied": sum(r["analysis"]["trim"]["tokens_saved"] for r in applied),
|
|
219
|
+
},
|
|
220
|
+
"requests": len(self.recent),
|
|
221
|
+
"analysed": len(analysed),
|
|
222
|
+
"analysis_errors": sum(1 for r in self.recent if r.get("analysis_error")),
|
|
223
|
+
"by_provider": dict(sorted(providers.items())),
|
|
224
|
+
"tokens_total": total,
|
|
225
|
+
"tokens_by_kind": by_kind,
|
|
226
|
+
"share_by_kind": {k: round(v / total, 3) if total else 0.0 for k, v in by_kind.items()},
|
|
227
|
+
"tool_result_tokens": by_result,
|
|
228
|
+
"attachments": attachments, # estimated provider tokens, forwarded unchanged
|
|
229
|
+
"billed_usage": usage_total,
|
|
230
|
+
"by_route": self._by_route(),
|
|
231
|
+
"tasks": _tasks(analysed),
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
def _by_route(self) -> dict:
|
|
235
|
+
"""Requests, tokens saved and billed usage per route (dual-plan demo)."""
|
|
236
|
+
out: dict[str, dict] = {}
|
|
237
|
+
for r in self.recent:
|
|
238
|
+
route = r.get("route") or r["provider"]
|
|
239
|
+
entry = out.setdefault(route, {
|
|
240
|
+
"requests": 0, "tokens": 0, "tokens_saved": 0,
|
|
241
|
+
"billed_usage": {"input_tokens": 0, "cache_read_tokens": 0,
|
|
242
|
+
"cache_write_tokens": 0, "output_tokens": 0},
|
|
243
|
+
})
|
|
244
|
+
entry["requests"] += 1
|
|
245
|
+
entry["tokens"] += _tokens(r)
|
|
246
|
+
entry["tokens_saved"] += _saved(r)
|
|
247
|
+
for field, value in (r.get("usage") or {}).items():
|
|
248
|
+
entry["billed_usage"][field] = entry["billed_usage"].get(field, 0) + value
|
|
249
|
+
return dict(sorted(out.items()))
|
|
250
|
+
|
|
251
|
+
def recent_feed(self, limit: int = 40) -> list[dict]:
|
|
252
|
+
"""The latest requests, newest first, numbers only (for the live page)."""
|
|
253
|
+
feed = []
|
|
254
|
+
for r in list(self.recent)[-limit:][::-1]:
|
|
255
|
+
trim = (r.get("analysis") or {}).get("trim") or {}
|
|
256
|
+
feed.append({
|
|
257
|
+
"ts": r["ts"],
|
|
258
|
+
"route": r.get("route") or r["provider"],
|
|
259
|
+
"model": (r.get("analysis") or {}).get("model", ""),
|
|
260
|
+
"status": r.get("status"),
|
|
261
|
+
"latency_ms": r.get("latency_ms"),
|
|
262
|
+
"tokens": _tokens(r),
|
|
263
|
+
"tokens_saved": _saved(r),
|
|
264
|
+
"applied": bool(r.get("trim_applied")),
|
|
265
|
+
"cache_event": trim.get("cache_event"),
|
|
266
|
+
"task": (r.get("analysis") or {}).get("task"),
|
|
267
|
+
"trims": [
|
|
268
|
+
{"how": d["how"], "before": d["before"], "after": d["after"]}
|
|
269
|
+
for d in trim.get("details", [])
|
|
270
|
+
],
|
|
271
|
+
"usage": r.get("usage"),
|
|
272
|
+
})
|
|
273
|
+
return feed
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _tasks(analysed: list[dict]) -> dict:
|
|
277
|
+
"""Task type of the newest typed message per request (labels and counts only)."""
|
|
278
|
+
tasks = [r["analysis"]["task"] for r in analysed if (r["analysis"].get("task") or {}).get("label")
|
|
279
|
+
or (r["analysis"].get("task") or {}).get("reason")]
|
|
280
|
+
return {"by_label": _count([t["label"] for t in tasks if t.get("label")]),
|
|
281
|
+
"by_reason": _count([t["reason"] for t in tasks if t.get("reason")]),
|
|
282
|
+
"briefed": sum(1 for t in tasks if t.get("briefed"))}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _tokens(record: dict) -> int:
|
|
286
|
+
return int((record.get("analysis") or {}).get("tokens_total", 0))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _saved(record: dict) -> int:
|
|
290
|
+
"""Tokens removed: what was sent less (trim mode) or what could be (shadow mode)."""
|
|
291
|
+
trim = (record.get("analysis") or {}).get("trim") or {}
|
|
292
|
+
return int(trim.get("tokens_saved", 0))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _count(values: list[str]) -> dict[str, int]:
|
|
296
|
+
out: dict[str, int] = {}
|
|
297
|
+
for v in values:
|
|
298
|
+
out[v] = out.get(v, 0) + 1
|
|
299
|
+
return dict(sorted(out.items()))
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _saved_by(trims: list[dict]) -> dict[str, int]:
|
|
303
|
+
"""Tokens saved per trimming method, summed over requests."""
|
|
304
|
+
out: dict[str, int] = {}
|
|
305
|
+
for t in trims:
|
|
306
|
+
for d in t.get("details", []):
|
|
307
|
+
out[d["how"]] = out.get(d["how"], 0) + d["before"] - d["after"]
|
|
308
|
+
return dict(sorted(out.items()))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def now_iso() -> str:
|
|
312
|
+
return datetime.now(UTC).isoformat(timespec="milliseconds")
|