tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,71 @@
1
+ """A fixed system note that tells the model how to treat TokenCut's markers (prompt enhancement).
2
+
3
+ Measured harms it targets: after an `omitted` marker the model re-reads the whole file, or
4
+ re-runs a command to see cut log lines (live run L3). The note is the same text on every
5
+ request and sits at the end of the system prompt, never in a user message, so it is cached
6
+ with the rest of the prefix (~0.1x price after the first request). Only agent requests (two or
7
+ more client tools) get it; side requests (titles, summaries) are left alone.
8
+ """
9
+
10
+ from tokencut.gateway.history import _tool_lists
11
+ from tokencut.gateway.virtual import MIN_CLIENT_TOOLS, _is_server_tool
12
+ from tokencut.tokens import count_tokens
13
+
14
+ SYSTEM_NOTE = (
15
+ "TokenCut note: tool output in this conversation may be shortened to save tokens. A line "
16
+ "like [TokenCut: lines A-B of path omitted ...] means those lines exist but were not sent; "
17
+ "if you need them, read only lines A-B (for example Read with offset/limit, or "
18
+ "sed -n 'A,Bp' path), not the whole file again. Shortened command output keeps the errors, "
19
+ "failures and final summary: do not re-run a command only to see the omitted lines. Prefer "
20
+ "reading files in line ranges, and keep test output short (for example pytest -q --tb=short)."
21
+ )
22
+ _SYSTEM_ROLES = ("system", "developer")
23
+
24
+
25
+ def add_system_note(provider: str, body: dict) -> dict | None:
26
+ """Add SYSTEM_NOTE to an agent request's system prompt, in place. Report entry or None."""
27
+ if not _is_agent_request(body):
28
+ return None
29
+ if provider == "anthropic":
30
+ system = body.get("system")
31
+ if isinstance(system, list):
32
+ if any(isinstance(b, dict) and b.get("text") == SYSTEM_NOTE for b in system):
33
+ return None
34
+ system.append({"type": "text", "text": SYSTEM_NOTE})
35
+ elif isinstance(system, str) and system:
36
+ if SYSTEM_NOTE in system:
37
+ return None
38
+ body["system"] = f"{system}\n\n{SYSTEM_NOTE}"
39
+ elif not system:
40
+ body["system"] = SYSTEM_NOTE
41
+ else:
42
+ return None
43
+ elif provider == "openai":
44
+ messages = body.get("messages")
45
+ if not isinstance(messages, list):
46
+ return None # not Chat Completions
47
+ if any(isinstance(m, dict) and m.get("content") == SYSTEM_NOTE for m in messages):
48
+ return None
49
+ lead = 0
50
+ while lead < len(messages) and isinstance(messages[lead], dict) \
51
+ and messages[lead].get("role") in _SYSTEM_ROLES:
52
+ lead += 1
53
+ role = messages[0]["role"] if lead else "system" # "developer" where the agent uses it
54
+ messages.insert(lead, {"role": role, "content": SYSTEM_NOTE})
55
+ elif provider == "gemini":
56
+ key = "system_instruction" if "system_instruction" in body else "systemInstruction"
57
+ system = body.setdefault(key, {})
58
+ parts = system.setdefault("parts", []) if isinstance(system, dict) else None
59
+ if not isinstance(parts, list) or {"text": SYSTEM_NOTE} in parts:
60
+ return None
61
+ parts.append({"text": SYSTEM_NOTE})
62
+ else:
63
+ return None
64
+ return {"segment": "system", "how": "system_note", "tool": "", "before": 0,
65
+ "after": count_tokens(SYSTEM_NOTE)}
66
+
67
+
68
+ def _is_agent_request(body: dict) -> bool:
69
+ client_tools = sum(1 for tools, _ in _tool_lists(body) for t in tools
70
+ if isinstance(t, dict) and not _is_server_tool(t))
71
+ return client_tools >= MIN_CLIENT_TOOLS
@@ -0,0 +1,214 @@
1
+ """Classify a user's request into a task type (prompt enhancement, docs/PROMPT_ROUTER_LABELS.md).
2
+
3
+ Cascade: cheap rules first (follow-ups like "yes", empty text), then a classifier on frozen
4
+ sentence embeddings (local, fastembed / ONNX on CPU). A prediction that is not confident
5
+ enough goes to `ambiguous`, and the user's message is then left unchanged: a wrong label
6
+ would give the agent wrong advice. Everything here is deterministic for the same text and
7
+ the same saved model, so a message gets the same label on every turn (prompt cache safe).
8
+ """
9
+
10
+ import re
11
+ from collections.abc import Callable
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+
16
+ AMBIGUOUS = "ambiguous"
17
+ STAGE1 = {
18
+ "bugfix": "change_code",
19
+ "feature": "change_code",
20
+ "refactor": "change_code",
21
+ "tests": "change_code",
22
+ "explain": "understand_code",
23
+ "review": "understand_code",
24
+ "design": "plan",
25
+ "run_action": "run_action",
26
+ "write_text": "write_text",
27
+ "followup": "followup",
28
+ AMBIGUOUS: AMBIGUOUS,
29
+ }
30
+ LABELS = tuple(STAGE1)
31
+ MODEL = "BAAI/bge-small-en-v1.5" # same model as the offline benchmark and the repo index
32
+ EMBED_BATCH = 8
33
+ MAX_CHARS = 2_000 # the start of a request says what is wanted; long pastes follow it
34
+
35
+ _FOLLOWUP_WORDS = frozenset(
36
+ ["y", "yes", "yep", "yeah", "yup", "ok", "okay", "k", "sure", "no", "nope", "continue", "go", "ahead", "on", "proceed", "do", "it", "that", "this", "looks", "look", "good", "great", "fine", "lgtm", "thanks", "thank", "you", "please", "option", "use", "the", "first", "second", "third", "one", "right", "correct", "agreed", "next", "sounds", "perfect"]
37
+ )
38
+ _WORD_RE = re.compile(r"[a-z]+|\d+")
39
+ MAX_FOLLOWUP_WORDS = 6
40
+
41
+
42
+ def rule_label(text: str) -> str | None:
43
+ """Label decided without a model, or None (then the classifier decides)."""
44
+ words = _WORD_RE.findall(text.lower())
45
+ if not words:
46
+ return AMBIGUOUS
47
+ if len(words) <= MAX_FOLLOWUP_WORDS and all(w in _FOLLOWUP_WORDS or w.isdigit() for w in words):
48
+ return "followup"
49
+ return None
50
+
51
+
52
+ def _unit(x: np.ndarray) -> np.ndarray:
53
+ x = np.asarray(x, dtype=np.float32)
54
+ return x / np.maximum(np.linalg.norm(x, axis=1, keepdims=True), 1e-12)
55
+
56
+
57
+ def _softmax(z: np.ndarray) -> np.ndarray:
58
+ z = z - z.max(axis=1, keepdims=True)
59
+ e = np.exp(z)
60
+ return e / e.sum(axis=1, keepdims=True)
61
+
62
+
63
+ class Prototypes:
64
+ """One mean vector per label; probability = softmax of scaled cosine similarity."""
65
+
66
+ def __init__(self, temperature: float = 20.0):
67
+ self.temperature = temperature
68
+
69
+ def fit(self, x: np.ndarray, y: list[str]) -> "Prototypes":
70
+ x = _unit(x)
71
+ self.labels = sorted(set(y))
72
+ self.centroids = _unit(np.stack([x[[t == lab for t in y]].mean(axis=0) for lab in self.labels]))
73
+ return self
74
+
75
+ def predict_proba(self, x: np.ndarray) -> np.ndarray:
76
+ return _softmax(self.temperature * _unit(x) @ self.centroids.T)
77
+
78
+
79
+ class KNN:
80
+ """Similarity-weighted vote of the k nearest training prompts."""
81
+
82
+ def __init__(self, k: int = 5, temperature: float = 20.0):
83
+ self.k, self.temperature = k, temperature
84
+
85
+ def fit(self, x: np.ndarray, y: list[str]) -> "KNN":
86
+ self.x = _unit(x)
87
+ self.labels = sorted(set(y))
88
+ self.y = np.array([self.labels.index(t) for t in y])
89
+ return self
90
+
91
+ def predict_proba(self, x: np.ndarray) -> np.ndarray:
92
+ sims = _unit(x) @ self.x.T
93
+ k = min(self.k, sims.shape[1])
94
+ top = np.argsort(-sims, axis=1, kind="stable")[:, :k]
95
+ out = np.full((len(x), len(self.labels)), 1e-6, dtype=np.float64)
96
+ for i, row in enumerate(top):
97
+ weights = np.exp(self.temperature * (sims[i, row] - sims[i, row].max()))
98
+ np.add.at(out[i], self.y[row], weights)
99
+ return out / out.sum(axis=1, keepdims=True)
100
+
101
+
102
+ class SoftmaxHead:
103
+ """Multinomial logistic regression on frozen embeddings (full-batch Adam, L2).
104
+
105
+ class_weight="balanced": each label counts as much as any other in the loss, however many
106
+ examples it has (weight = n / (labels * count)), so rare labels are not drowned out."""
107
+
108
+ def __init__(self, l2: float = 1e-3, epochs: int = 500, lr: float = 0.05,
109
+ class_weight: str | None = None):
110
+ self.l2, self.epochs, self.lr, self.class_weight = l2, epochs, lr, class_weight
111
+
112
+ def fit(self, x: np.ndarray, y: list[str]) -> "SoftmaxHead":
113
+ x = _unit(x).astype(np.float64)
114
+ self.labels = sorted(set(y))
115
+ target = np.zeros((len(y), len(self.labels)))
116
+ target[np.arange(len(y)), [self.labels.index(t) for t in y]] = 1
117
+ weight = np.ones((len(y), 1))
118
+ if self.class_weight == "balanced":
119
+ counts = target.sum(axis=0)
120
+ weight = (len(y) / (len(self.labels) * counts))[[self.labels.index(t) for t in y]][:, None]
121
+ w = np.zeros((x.shape[1], len(self.labels)))
122
+ b = np.zeros(len(self.labels))
123
+ m = [np.zeros_like(w), np.zeros_like(b)]
124
+ v = [np.zeros_like(w), np.zeros_like(b)]
125
+ for step in range(1, self.epochs + 1):
126
+ grad = weight * (_softmax(x @ w + b) - target) / len(y)
127
+ grads = [x.T @ grad + self.l2 * w, grad.sum(axis=0)]
128
+ for j, (param, g) in enumerate(zip((w, b), grads)):
129
+ m[j] = 0.9 * m[j] + 0.1 * g
130
+ v[j] = 0.999 * v[j] + 0.001 * g * g
131
+ param -= self.lr * (m[j] / (1 - 0.9**step)) / (np.sqrt(v[j] / (1 - 0.999**step)) + 1e-8)
132
+ self.w, self.b = w.astype(np.float32), b.astype(np.float32)
133
+ return self
134
+
135
+ def predict_proba(self, x: np.ndarray) -> np.ndarray:
136
+ return _softmax(_unit(x) @ self.w + self.b)
137
+
138
+ def save(self, path: Path) -> None:
139
+ np.savez(path, w=self.w, b=self.b, labels=np.array(self.labels))
140
+
141
+ @classmethod
142
+ def load(cls, path: Path) -> "SoftmaxHead":
143
+ data = np.load(path)
144
+ head = cls()
145
+ head.w, head.b, head.labels = data["w"], data["b"], [str(s) for s in data["labels"]]
146
+ return head
147
+
148
+
149
+ class TwoStage:
150
+ """Broad group first (STAGE1), then the label inside the group: p = p(group) * p(label|group)."""
151
+
152
+ def __init__(self, make: Callable[[], object] = SoftmaxHead):
153
+ self.make = make
154
+
155
+ def fit(self, x: np.ndarray, y: list[str]) -> "TwoStage":
156
+ self.labels = sorted(set(y))
157
+ groups = [STAGE1[t] for t in y]
158
+ self.stage1 = self.make().fit(x, groups)
159
+ self.stage2 = {}
160
+ for group in self.stage1.labels:
161
+ rows = [i for i, g in enumerate(groups) if g == group]
162
+ if len({y[i] for i in rows}) > 1:
163
+ self.stage2[group] = self.make().fit(x[rows], [y[i] for i in rows])
164
+ return self
165
+
166
+ def predict_proba(self, x: np.ndarray) -> np.ndarray:
167
+ p1 = self.stage1.predict_proba(x)
168
+ out = np.zeros((len(x), len(self.labels)))
169
+ for g, group in enumerate(self.stage1.labels):
170
+ if group in self.stage2:
171
+ inner = self.stage2[group]
172
+ p2 = inner.predict_proba(x)
173
+ for j, label in enumerate(inner.labels):
174
+ out[:, self.labels.index(label)] = p1[:, g] * p2[:, j]
175
+ else:
176
+ label = next(t for t in self.labels if STAGE1[t] == group)
177
+ out[:, self.labels.index(label)] = p1[:, g]
178
+ return out
179
+
180
+
181
+ def fastembed_embedder(cache_dir: Path, model: str = MODEL) -> Callable[[list[str]], np.ndarray]:
182
+ from fastembed import TextEmbedding # optional dependency, imported only when used
183
+
184
+ encoder = TextEmbedding(model, cache_dir=str(cache_dir / "models"))
185
+ # Small batches: a long-context model (8k tokens) embedding 256 long prompts at once
186
+ # runs a 16 GB laptop out of memory.
187
+ return lambda texts: np.stack([np.asarray(v, dtype=np.float32)
188
+ for v in encoder.embed(texts, batch_size=EMBED_BATCH)])
189
+
190
+
191
+ class PromptRouter:
192
+ """rules -> classifier -> `ambiguous` when the top label is not clearly ahead."""
193
+
194
+ def __init__(self, model, embed: Callable[[list[str]], np.ndarray],
195
+ min_prob: float = 0.6, min_margin: float = 0.15):
196
+ self.model, self.embed = model, embed
197
+ self.min_prob, self.min_margin = min_prob, min_margin
198
+ self._cache: dict[str, tuple[str, float]] = {}
199
+
200
+ def classify(self, text: str) -> tuple[str, float]:
201
+ ruled = rule_label(text)
202
+ if ruled is not None:
203
+ return ruled, 1.0
204
+ key = text[:MAX_CHARS]
205
+ if key not in self._cache:
206
+ probs = self.model.predict_proba(self.embed([key]))[0]
207
+ order = np.argsort(-probs, kind="stable")
208
+ top = float(probs[order[0]])
209
+ second = float(probs[order[1]]) if len(order) > 1 else 0.0
210
+ label = self.model.labels[order[0]]
211
+ if top < self.min_prob or top - second < self.min_margin:
212
+ label = AMBIGUOUS
213
+ self._cache[key] = (label, top)
214
+ return self._cache[key]
@@ -0,0 +1,312 @@
1
+ """Gateway measurements: what each request contains, and what the provider billed.
2
+
3
+ Only numbers are logged (token counts per segment kind, usage), never prompt text or
4
+ headers, so the log is safe to keep. Full request bodies are saved only when capture is
5
+ switched on, for building parser test fixtures.
6
+ """
7
+
8
+ import copy
9
+ import dataclasses
10
+ import json
11
+ import logging
12
+ import time
13
+ from collections import deque
14
+ from datetime import UTC, datetime
15
+ from pathlib import Path
16
+
17
+ from tokencut.contracts import Segment, SegmentKind, ToolResultKind
18
+ from tokencut.gateway.adapters import anthropic, gemini, openai
19
+ from tokencut.gateway.adapters.common import lookup
20
+ from tokencut.gateway.brief import add_task_briefs
21
+ from tokencut.gateway.history import CacheEventConfig, trim_with_cache_events
22
+ from tokencut.gateway.media import adjust_images, split_pdfs, with_media_tokens
23
+ from tokencut.gateway.note import add_system_note
24
+ from tokencut.gateway.session import SessionStore
25
+ from tokencut.gateway.trimmer import TrimConfig, trim_request
26
+ from tokencut.gateway.virtual import VirtualConfig, finish_tools, offers_tools, resolve
27
+
28
+ log = logging.getLogger(__name__)
29
+ ADAPTERS = {"anthropic": anthropic, "openai": openai, "gemini": gemini}
30
+ RECENT_RECORDS = 1000
31
+
32
+
33
+ def analyse(
34
+ provider: str,
35
+ path: str,
36
+ body: bytes,
37
+ trim_config: TrimConfig | None = None,
38
+ session: tuple[SessionStore, CacheEventConfig] | None = None,
39
+ now: float | None = None,
40
+ virtual: VirtualConfig | None = None,
41
+ router=None,
42
+ ) -> tuple[dict, bytes | None]:
43
+ """Split a request into segments and count tokens per kind; with a TrimConfig, also
44
+ work out the cache-safe trim and the gateway's own tools (virtual.py). Returns
45
+ (analysis, body bytes to send or None if unchanged).
46
+ Raises on bad input (the caller then forwards the original body)."""
47
+ start = time.perf_counter()
48
+ data = json.loads(body)
49
+ adapter = ADAPTERS[provider]
50
+ answered = [] # the gateway's answers to its own tools (replace the agent's errors)
51
+ if trim_config is not None and virtual is not None:
52
+ answered = resolve(provider, data, virtual.store)
53
+ model = adapter.model_of(data, path)
54
+ # attachments are forwarded unchanged; their estimated provider tokens are counted
55
+ segments, attachments = with_media_tokens(provider, model, data, adapter.parse(data))
56
+ analysis = {
57
+ "model": model,
58
+ "stream": bool(data.get("stream")) or "streamGenerateContent" in path,
59
+ "segments": len(segments),
60
+ **token_breakdown(segments),
61
+ "media": attachments,
62
+ }
63
+ new_bytes = None
64
+ if trim_config is not None:
65
+ trim_start = time.perf_counter()
66
+ event = None
67
+ ccr = virtual is not None and virtual.ccr and offers_tools(provider, data)
68
+ if ccr:
69
+ trim_config = dataclasses.replace(trim_config, ccr=True)
70
+ if session is not None: # cache-event trimming (remembers decisions per conversation)
71
+ store, events_config = session
72
+ new_body, report, event = trim_with_cache_events(
73
+ provider, analysis["model"], data, segments, trim_config, events_config, store,
74
+ now=now, total_tokens=analysis["tokens_total"],
75
+ )
76
+ else:
77
+ new_body, report = trim_request(data, segments, trim_config)
78
+ # Attachments (steps 5-7). Applied last and in descending path order: an image may
79
+ # gain a note and a PDF may become two blocks, which shifts later list positions.
80
+ done = {d["segment"] for d in report.trimmed} # e.g. a duplicate is already a note
81
+ replacements, entries = adjust_images(provider, model, data, segments,
82
+ trim_config.media_user, trim_config.resize_images,
83
+ trim_config.trim_borders)
84
+ if trim_config.pdf_text_pages:
85
+ pdf_changes, pdf_entries = split_pdfs(provider, data, segments, trim_config.media_user)
86
+ replacements, entries = replacements + pdf_changes, entries + pdf_entries
87
+ pending = [(p, b, e) for (p, b), e in zip(replacements, entries, strict=True)
88
+ if e["segment"] not in done]
89
+ for block_path, blocks, entry in sorted(pending, key=lambda c: tuple(c[0]), reverse=True):
90
+ if new_body is None:
91
+ new_body = copy.deepcopy(data)
92
+ parent = lookup(new_body, block_path[:-1])
93
+ parent[block_path[-1]:block_path[-1] + 1] = blocks
94
+ report.tokens_before += entry["before"]
95
+ report.tokens_after += entry["after"]
96
+ report.trimmed.append(entry)
97
+ if virtual is not None:
98
+ if ccr: # keep every original we shortened (fallback if it leaves the request)
99
+ shortened = {d["segment"] for d in report.trimmed}
100
+ for seg in segments:
101
+ if seg.id in shortened and seg.text: # attachments have no text to keep
102
+ virtual.store.put(seg.text)
103
+ out = new_body if new_body is not None else data
104
+ extra = answered + finish_tools(provider, out, ccr, virtual.lazy_tools,
105
+ virtual.core_tools)
106
+ for entry in extra:
107
+ report.tokens_before += entry["before"]
108
+ report.tokens_after += entry["after"]
109
+ report.trimmed.append(entry)
110
+ if extra:
111
+ new_body = out
112
+ if trim_config.prompt_brief and router is not None: # before the note (it shifts paths)
113
+ out = new_body if new_body is not None else copy.deepcopy(data)
114
+ try:
115
+ entries, analysis["task"] = add_task_briefs(provider, out, segments, router,
116
+ trim_config.brief_threshold)
117
+ except Exception as exc: # fail open: no briefs, the request still goes out
118
+ log.warning("task briefs failed", exc_info=True)
119
+ entries, analysis["task"] = [], {"error": type(exc).__name__}
120
+ for entry in entries:
121
+ report.tokens_after += entry["after"]
122
+ report.trimmed.append(entry)
123
+ if entries:
124
+ new_body = out
125
+ if trim_config.system_note: # last: it may insert a message (OpenAI), shifting paths
126
+ out = new_body if new_body is not None else copy.deepcopy(data)
127
+ entry = add_system_note(provider, out)
128
+ if entry is not None:
129
+ report.tokens_after += entry["after"]
130
+ report.trimmed.append(entry)
131
+ new_body = out
132
+ analysis["trim"] = {
133
+ **report.as_dict(),
134
+ "tier": trim_config.tier.value,
135
+ "cache_event": event,
136
+ "trim_ms": round((time.perf_counter() - trim_start) * 1000, 1),
137
+ }
138
+ if new_body is not None:
139
+ new_bytes = json.dumps(new_body, ensure_ascii=False, separators=(",", ":")).encode()
140
+ analysis["analysis_ms"] = round((time.perf_counter() - start) * 1000, 1)
141
+ return analysis, new_bytes
142
+
143
+
144
+ def token_breakdown(segments: list[Segment]) -> dict:
145
+ by_kind = {kind.value: 0 for kind in SegmentKind}
146
+ by_result = {kind.value: 0 for kind in ToolResultKind}
147
+ for s in segments:
148
+ by_kind[s.kind.value] += s.token_count
149
+ if s.result_kind is not None:
150
+ by_result[s.result_kind.value] += s.token_count
151
+ return {
152
+ "tokens_total": sum(by_kind.values()),
153
+ "tokens_by_kind": by_kind,
154
+ "tool_result_tokens": by_result,
155
+ }
156
+
157
+
158
+ class RequestLog:
159
+ def __init__(self, log_dir: Path | None, capture: bool = False):
160
+ self.log_dir = log_dir
161
+ self.capture = capture
162
+ self.recent: deque[dict] = deque(maxlen=RECENT_RECORDS)
163
+
164
+ def add(self, record: dict) -> None:
165
+ self.recent.append(record)
166
+ if self.log_dir is None:
167
+ return
168
+ self.log_dir.mkdir(parents=True, exist_ok=True)
169
+ day = record["ts"][:10]
170
+ with (self.log_dir / f"requests-{day}.jsonl").open("a", encoding="utf-8") as f:
171
+ f.write(json.dumps(record) + "\n")
172
+
173
+ def save_capture(self, provider: str, body: bytes) -> None:
174
+ """Keep a full request body (local only; contains the user's code and prompts)."""
175
+ if not self.capture or self.log_dir is None:
176
+ return
177
+ folder = self.log_dir / "captures"
178
+ folder.mkdir(parents=True, exist_ok=True)
179
+ stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%f")
180
+ (folder / f"{stamp}_{provider}.json").write_bytes(body)
181
+
182
+ def summary(self) -> dict:
183
+ """Totals over the recent requests, for GET /tokencut/stats."""
184
+ analysed = [r for r in self.recent if r.get("analysis")]
185
+ by_kind = {kind.value: 0 for kind in SegmentKind}
186
+ by_result = {kind.value: 0 for kind in ToolResultKind}
187
+ usage_total = {
188
+ "input_tokens": 0,
189
+ "cache_read_tokens": 0,
190
+ "cache_write_tokens": 0,
191
+ "output_tokens": 0,
192
+ }
193
+ providers: dict[str, int] = {}
194
+ for r in self.recent:
195
+ providers[r["provider"]] = providers.get(r["provider"], 0) + 1
196
+ for field, value in (r.get("usage") or {}).items():
197
+ usage_total[field] = usage_total.get(field, 0) + value
198
+ for r in analysed:
199
+ for kind, n in r["analysis"]["tokens_by_kind"].items():
200
+ by_kind[kind] += n
201
+ for kind, n in r["analysis"]["tool_result_tokens"].items():
202
+ by_result[kind] += n
203
+ attachments = {"images": 0, "documents": 0, "unknown": 0, "tokens": 0}
204
+ for r in analysed:
205
+ for field, value in (r["analysis"].get("media") or {}).items():
206
+ attachments[field] = attachments.get(field, 0) + value
207
+ total = sum(by_kind.values())
208
+ trims = [r["analysis"]["trim"] for r in analysed if r["analysis"].get("trim")]
209
+ applied = [r for r in analysed if r.get("trim_applied")]
210
+ return {
211
+ "trimming": {
212
+ "cache_events": _count(
213
+ [t["cache_event"] for t in trims if t.get("cache_event")]
214
+ ),
215
+ "saved_by": _saved_by(trims),
216
+ "requests_with_trims": sum(1 for t in trims if t["segments_trimmed"]),
217
+ "tokens_saved_possible": sum(t["tokens_saved"] for t in trims),
218
+ "tokens_saved_applied": sum(r["analysis"]["trim"]["tokens_saved"] for r in applied),
219
+ },
220
+ "requests": len(self.recent),
221
+ "analysed": len(analysed),
222
+ "analysis_errors": sum(1 for r in self.recent if r.get("analysis_error")),
223
+ "by_provider": dict(sorted(providers.items())),
224
+ "tokens_total": total,
225
+ "tokens_by_kind": by_kind,
226
+ "share_by_kind": {k: round(v / total, 3) if total else 0.0 for k, v in by_kind.items()},
227
+ "tool_result_tokens": by_result,
228
+ "attachments": attachments, # estimated provider tokens, forwarded unchanged
229
+ "billed_usage": usage_total,
230
+ "by_route": self._by_route(),
231
+ "tasks": _tasks(analysed),
232
+ }
233
+
234
+ def _by_route(self) -> dict:
235
+ """Requests, tokens saved and billed usage per route (dual-plan demo)."""
236
+ out: dict[str, dict] = {}
237
+ for r in self.recent:
238
+ route = r.get("route") or r["provider"]
239
+ entry = out.setdefault(route, {
240
+ "requests": 0, "tokens": 0, "tokens_saved": 0,
241
+ "billed_usage": {"input_tokens": 0, "cache_read_tokens": 0,
242
+ "cache_write_tokens": 0, "output_tokens": 0},
243
+ })
244
+ entry["requests"] += 1
245
+ entry["tokens"] += _tokens(r)
246
+ entry["tokens_saved"] += _saved(r)
247
+ for field, value in (r.get("usage") or {}).items():
248
+ entry["billed_usage"][field] = entry["billed_usage"].get(field, 0) + value
249
+ return dict(sorted(out.items()))
250
+
251
+ def recent_feed(self, limit: int = 40) -> list[dict]:
252
+ """The latest requests, newest first, numbers only (for the live page)."""
253
+ feed = []
254
+ for r in list(self.recent)[-limit:][::-1]:
255
+ trim = (r.get("analysis") or {}).get("trim") or {}
256
+ feed.append({
257
+ "ts": r["ts"],
258
+ "route": r.get("route") or r["provider"],
259
+ "model": (r.get("analysis") or {}).get("model", ""),
260
+ "status": r.get("status"),
261
+ "latency_ms": r.get("latency_ms"),
262
+ "tokens": _tokens(r),
263
+ "tokens_saved": _saved(r),
264
+ "applied": bool(r.get("trim_applied")),
265
+ "cache_event": trim.get("cache_event"),
266
+ "task": (r.get("analysis") or {}).get("task"),
267
+ "trims": [
268
+ {"how": d["how"], "before": d["before"], "after": d["after"]}
269
+ for d in trim.get("details", [])
270
+ ],
271
+ "usage": r.get("usage"),
272
+ })
273
+ return feed
274
+
275
+
276
+ def _tasks(analysed: list[dict]) -> dict:
277
+ """Task type of the newest typed message per request (labels and counts only)."""
278
+ tasks = [r["analysis"]["task"] for r in analysed if (r["analysis"].get("task") or {}).get("label")
279
+ or (r["analysis"].get("task") or {}).get("reason")]
280
+ return {"by_label": _count([t["label"] for t in tasks if t.get("label")]),
281
+ "by_reason": _count([t["reason"] for t in tasks if t.get("reason")]),
282
+ "briefed": sum(1 for t in tasks if t.get("briefed"))}
283
+
284
+
285
+ def _tokens(record: dict) -> int:
286
+ return int((record.get("analysis") or {}).get("tokens_total", 0))
287
+
288
+
289
+ def _saved(record: dict) -> int:
290
+ """Tokens removed: what was sent less (trim mode) or what could be (shadow mode)."""
291
+ trim = (record.get("analysis") or {}).get("trim") or {}
292
+ return int(trim.get("tokens_saved", 0))
293
+
294
+
295
+ def _count(values: list[str]) -> dict[str, int]:
296
+ out: dict[str, int] = {}
297
+ for v in values:
298
+ out[v] = out.get(v, 0) + 1
299
+ return dict(sorted(out.items()))
300
+
301
+
302
+ def _saved_by(trims: list[dict]) -> dict[str, int]:
303
+ """Tokens saved per trimming method, summed over requests."""
304
+ out: dict[str, int] = {}
305
+ for t in trims:
306
+ for d in t.get("details", []):
307
+ out[d["how"]] = out.get(d["how"], 0) + d["before"] - d["after"]
308
+ return dict(sorted(out.items()))
309
+
310
+
311
+ def now_iso() -> str:
312
+ return datetime.now(UTC).isoformat(timespec="milliseconds")