tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,825 @@
1
+ """Attachment token estimates: images and documents (docs/ATTACHMENTS_PLAN.md, step 1).
2
+
3
+ Attachments are forwarded unchanged (`SegmentKind.MEDIA`). Their text is empty, so until now
4
+ they counted as 0 tokens. This module estimates what the provider bills for them, from the
5
+ formulas the providers publish (read Nov 2026, see the plan's sources), so stats and later
6
+ cost decisions see them. Estimates are in the provider's own tokens, not proxy tokens.
7
+
8
+ Only headers are read (no image library). Unknown sizes (an image behind a URL or a file id)
9
+ count as 0 and are reported as `unknown`.
10
+ """
11
+
12
+ import base64
13
+ import binascii
14
+ import copy
15
+ import hashlib
16
+ import io
17
+ import math
18
+ import re
19
+ import struct
20
+ from typing import Any
21
+
22
+ from PIL import Image, ImageChops
23
+ from pypdf import PdfReader, PdfWriter
24
+ from pypdf.errors import PyPdfError
25
+
26
+ from tokencut.contracts import Segment, SegmentKind
27
+ from tokencut.gateway.adapters.common import lookup
28
+ from tokencut.gateway.routing import DEFAULT_GPT_PREFIXES
29
+ from tokencut.tokens import count_tokens
30
+
31
+ HEAD_B64_CHARS = 1 << 18 # 256k base64 characters (~192 KB): enough to reach an image header
32
+ PDF_TEXT_TOKENS_PER_PAGE = 2_250 # Claude: "typically 1,500-3,000" text tokens per page (midpoint)
33
+ _PDF_CACHE_MAX = 64
34
+ _pdf_cache: dict[tuple[int, str, str], int | None] = {}
35
+
36
+ # ---------- reading sizes ----------
37
+
38
+ _JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
39
+
40
+
41
+ def image_size(raw: bytes) -> tuple[int, int] | None:
42
+ """(width, height) from a PNG / JPEG / GIF / WebP header, or None."""
43
+ try:
44
+ if raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR":
45
+ return struct.unpack(">II", raw[16:24])
46
+ if raw[:6] in (b"GIF87a", b"GIF89a"):
47
+ return struct.unpack("<HH", raw[6:10])
48
+ if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP":
49
+ return _webp_size(raw)
50
+ if raw[:2] == b"\xff\xd8":
51
+ return _jpeg_size(raw)
52
+ except struct.error:
53
+ return None
54
+ return None
55
+
56
+
57
+ def _webp_size(raw: bytes) -> tuple[int, int] | None:
58
+ chunk = raw[12:16]
59
+ if chunk == b"VP8X":
60
+ return 1 + int.from_bytes(raw[24:27], "little"), 1 + int.from_bytes(raw[27:30], "little")
61
+ if chunk == b"VP8L":
62
+ bits = int.from_bytes(raw[21:25], "little")
63
+ return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
64
+ if chunk == b"VP8 ":
65
+ w, h = struct.unpack("<HH", raw[26:30])
66
+ return w & 0x3FFF, h & 0x3FFF
67
+ return None
68
+
69
+
70
+ def _jpeg_size(raw: bytes) -> tuple[int, int] | None:
71
+ i = 2
72
+ while i + 9 <= len(raw):
73
+ if raw[i] != 0xFF:
74
+ i += 1
75
+ continue
76
+ marker = raw[i + 1]
77
+ if marker in _JPEG_SOF:
78
+ h, w = struct.unpack(">HH", raw[i + 5:i + 9])
79
+ return w, h
80
+ if marker == 0xFF: # fill byte
81
+ i += 1
82
+ elif marker in (0x01, 0xD8) or 0xD0 <= marker <= 0xD7: # markers without a length
83
+ i += 2
84
+ else:
85
+ i += 2 + struct.unpack(">H", raw[i + 2:i + 4])[0]
86
+ return None
87
+
88
+
89
+ def pdf_pages(raw: bytes) -> int | None:
90
+ """Page count from the page objects, else from the page tree's /Count, else (page objects
91
+ inside compressed object streams, PDF 1.5+) from the PDF parser."""
92
+ if not raw.startswith(b"%PDF"):
93
+ return None
94
+ pages = len(re.findall(rb"/Type\s*/Page(?![A-Za-z])", raw))
95
+ if pages:
96
+ return pages
97
+ counts = [int(c) for c in re.findall(rb"/Type\s*/Pages\b[^>]*?/Count\s+(\d+)", raw)]
98
+ if counts:
99
+ return max(counts)
100
+ try:
101
+ return len(PdfReader(io.BytesIO(raw)).pages) or None
102
+ except _PDF_ERRORS:
103
+ return None
104
+
105
+
106
+ def _decode(data: str, limit: int | None = None) -> bytes:
107
+ part = data if limit is None else data[:limit - limit % 4]
108
+ try:
109
+ return base64.b64decode(part, validate=False)
110
+ except (binascii.Error, ValueError):
111
+ return b""
112
+
113
+
114
+ def _pdf_pages_b64(data: str) -> int | None:
115
+ """Whole document decoded once per distinct PDF (the same PDF is re-sent every turn)."""
116
+ key = (len(data), data[:2048], data[-2048:])
117
+ if key not in _pdf_cache:
118
+ if len(_pdf_cache) >= _PDF_CACHE_MAX:
119
+ _pdf_cache.pop(next(iter(_pdf_cache)))
120
+ _pdf_cache[key] = pdf_pages(_decode(data))
121
+ return _pdf_cache[key]
122
+
123
+
124
+ # ---------- provider formulas ----------
125
+
126
+
127
+ def claude_high_res(model: str) -> bool:
128
+ """Claude 4.7 and later use the high-resolution image tier."""
129
+ m = re.search(r"claude-(?:[a-z]+-)?(\d+)-(\d{1,2})(?!\d)", model) or re.search(
130
+ r"claude-(?:[a-z]+-)?(\d+)", model)
131
+ if not m:
132
+ return False
133
+ major = int(m.group(1))
134
+ minor = int(m.group(2)) if m.lastindex and m.lastindex >= 2 else 0
135
+ return (major, minor) >= (4, 7)
136
+
137
+
138
+ def _claude_tokens(w: int, h: int) -> int:
139
+ return math.ceil(w / 28) * math.ceil(h / 28)
140
+
141
+
142
+ def claude_image_tokens(w: int, h: int, model: str) -> int:
143
+ """ceil(w/28) x ceil(h/28); larger images are shrunk (aspect kept) to the tier's limits."""
144
+ edge, cap = (2576, 4784) if claude_high_res(model) else (1568, 1568)
145
+ if max(w, h) <= edge and _claude_tokens(w, h) <= cap:
146
+ return _claude_tokens(w, h)
147
+ long_side, short_side = max(w, h), min(w, h)
148
+ side = min(edge, long_side)
149
+ while side > 1:
150
+ other = max(1, round(side * short_side / long_side))
151
+ if _claude_tokens(side, other) <= cap:
152
+ return _claude_tokens(side, other)
153
+ side -= 1
154
+ return 1
155
+
156
+
157
+ # (base, per tile) for tile-priced OpenAI models; first matching prefix wins
158
+ _OPENAI_TILES = (("gpt-4o-mini", (2833, 5667)), ("gpt-4.1-mini", None), ("gpt-4.1-nano", None),
159
+ ("gpt-4o", (85, 170)), ("gpt-4.1", (85, 170)), ("gpt-5.1", (70, 140)))
160
+ _OPENAI_PATCH_MULTIPLIER = (("gpt-4.1-mini", 1.62), ("gpt-5-nano", 1.5))
161
+ _OPENAI_DEFAULT_MULTIPLIER = 1.2 # gpt-5.2 / 5.4 / 5.5 in the guide
162
+ _OPENAI_MAX_SIDE, _OPENAI_MAX_PATCHES = 2048, 2500
163
+
164
+
165
+ def _fit(w: int, h: int, box: int) -> tuple[int, int]:
166
+ scale = min(1.0, box / max(w, h))
167
+ return max(1, round(w * scale)), max(1, round(h * scale))
168
+
169
+
170
+ def openai_image_tokens(w: int, h: int, model: str, detail: str = "auto") -> int:
171
+ """Tile models: base + per-tile x 512-px tiles. Patch models: 32-px patches x multiplier."""
172
+ name = model.lower()
173
+ tiles = next((t for prefix, t in _OPENAI_TILES if name.startswith(prefix)), None)
174
+ if tiles is not None:
175
+ base, per_tile = tiles
176
+ if detail == "low":
177
+ return base
178
+ w, h = _fit(w, h, 2048)
179
+ if min(w, h) > 768:
180
+ scale = 768 / min(w, h)
181
+ w, h = round(w * scale), round(h * scale)
182
+ return base + per_tile * math.ceil(w / 512) * math.ceil(h / 512)
183
+ mult = next((m for prefix, m in _OPENAI_PATCH_MULTIPLIER if name.startswith(prefix)),
184
+ _OPENAI_DEFAULT_MULTIPLIER)
185
+ if detail == "low":
186
+ w, h = _fit(w, h, 512)
187
+ elif detail != "original":
188
+ w, h = _fit(w, h, _OPENAI_MAX_SIDE)
189
+ while math.ceil(w / 32) * math.ceil(h / 32) > _OPENAI_MAX_PATCHES:
190
+ w, h = _fit(w, h, max(w, h) - 1)
191
+ return math.ceil(math.ceil(w / 32) * math.ceil(h / 32) * mult)
192
+
193
+
194
+ _GEMINI_IMAGE = {"LOW": 280, "MEDIUM": 560, "HIGH": 1120, "ULTRA_HIGH": 2240}
195
+ _GEMINI_PDF_PAGE = {"LOW": 280, "MEDIUM": 560, "HIGH": 1120}
196
+
197
+
198
+ def _gemini_new(model: str) -> bool:
199
+ m = re.search(r"gemini-(\d+)", model)
200
+ return m is None or int(m.group(1)) >= 3 # unknown: assume a current model
201
+
202
+
203
+ def _resolution(value: str) -> str:
204
+ return value.upper().removeprefix("MEDIA_RESOLUTION_")
205
+
206
+
207
+ def gemini_image_tokens(w: int, h: int, model: str, resolution: str = "") -> int:
208
+ if _gemini_new(model):
209
+ return _GEMINI_IMAGE.get(_resolution(resolution), 1120)
210
+ if w <= 384 and h <= 384:
211
+ return 258
212
+ unit = max(1, math.floor(min(w, h) / 1.5))
213
+ return 258 * math.ceil(w / unit) * math.ceil(h / unit)
214
+
215
+
216
+ def gemini_pdf_page_tokens(model: str, resolution: str = "") -> int:
217
+ if _gemini_new(model):
218
+ return _GEMINI_PDF_PAGE.get(_resolution(resolution), 560)
219
+ return 258
220
+
221
+
222
+ # ---------- attachments inside requests ----------
223
+
224
+
225
+ def _data_url(url: str) -> tuple[str, str] | None:
226
+ m = re.match(r"data:([^;,]+)[^,]*;base64,", url)
227
+ return (m.group(1), url[m.end():]) if m else None
228
+
229
+
230
+ def _estimate(provider: str, model: str, block: Any, body: dict) -> tuple[str, int | None] | None:
231
+ """(kind "image"/"document", tokens or None if unknown) for an attachment block, else None."""
232
+ if not isinstance(block, dict):
233
+ return None
234
+ if provider == "anthropic" and block.get("type") in ("image", "document"):
235
+ kind = block["type"]
236
+ source = block.get("source") or {}
237
+ if source.get("type") == "text":
238
+ return kind, count_tokens(str(source.get("data", "")))
239
+ if source.get("type") != "base64":
240
+ return kind, None
241
+ return kind, _from_bytes(provider, model, str(source.get("media_type", "")),
242
+ str(source.get("data", "")), "")
243
+ if provider == "openai":
244
+ if block.get("type") == "image_url":
245
+ image = block.get("image_url") or {}
246
+ parsed = _data_url(str(image.get("url", "")))
247
+ if parsed is None:
248
+ return "image", None
249
+ return "image", _from_bytes(provider, model, parsed[0], parsed[1],
250
+ str(image.get("detail", "auto")))
251
+ if block.get("type") == "file":
252
+ parsed = _data_url(str((block.get("file") or {}).get("file_data", "")))
253
+ if parsed is None:
254
+ return "document", None
255
+ return "document", _from_bytes(provider, model, parsed[0], parsed[1], "")
256
+ return None
257
+ if provider == "gemini":
258
+ inline = block.get("inlineData") or block.get("inline_data")
259
+ file_data = block.get("fileData") or block.get("file_data")
260
+ if isinstance(file_data, dict):
261
+ mime = str(file_data.get("mimeType") or file_data.get("mime_type") or "")
262
+ return ("image" if mime.startswith("image/") else "document"), None
263
+ if not isinstance(inline, dict):
264
+ return None
265
+ mime = str(inline.get("mimeType") or inline.get("mime_type") or "")
266
+ config = body.get("generationConfig") or body.get("generation_config") or {}
267
+ resolution = str(config.get("mediaResolution") or config.get("media_resolution") or "")
268
+ kind = "image" if mime.startswith("image/") else "document"
269
+ return kind, _from_bytes(provider, model, mime, str(inline.get("data", "")), resolution)
270
+ return None
271
+
272
+
273
+ def pricing_provider(provider: str, model: str) -> str:
274
+ """Whose image / PDF prices apply. Claude Code sends GPT models in Claude's format when
275
+ TokenCut routes them through LiteLLM to the ChatGPT plan: those are priced as OpenAI."""
276
+ if provider == "anthropic" and model.lower().startswith(DEFAULT_GPT_PREFIXES):
277
+ return "openai"
278
+ return provider
279
+
280
+
281
+ def _from_bytes(provider: str, model: str, mime: str, data: str, option: str) -> int | None:
282
+ provider = pricing_provider(provider, model)
283
+ if mime.startswith("image/"):
284
+ size = image_size(_decode(data, HEAD_B64_CHARS))
285
+ if size is None:
286
+ return None
287
+ w, h = size
288
+ if provider == "anthropic":
289
+ return claude_image_tokens(w, h, model)
290
+ if provider == "openai":
291
+ return openai_image_tokens(w, h, model, option or "auto")
292
+ return gemini_image_tokens(w, h, model, option)
293
+ if mime == "application/pdf":
294
+ pages = _pdf_pages_b64(data)
295
+ if pages is None:
296
+ return None
297
+ if provider == "gemini":
298
+ return pages * gemini_pdf_page_tokens(model, option)
299
+ return pages * PDF_TEXT_TOKENS_PER_PAGE # text side only: page images come on top
300
+ if mime.startswith("text/"):
301
+ return count_tokens(_decode(data).decode("utf-8", errors="replace"))
302
+ return None
303
+
304
+
305
+ def attachment_key(block: Any) -> str | None:
306
+ """Content hash of an inline attachment (Anthropic base64 block, Gemini inlineData)."""
307
+ if not isinstance(block, dict):
308
+ return None
309
+ data = None
310
+ if block.get("type") in ("image", "document"):
311
+ source = block.get("source") or {}
312
+ if source.get("type") == "base64":
313
+ data = source.get("data")
314
+ else:
315
+ inline = block.get("inlineData") or block.get("inline_data")
316
+ if isinstance(inline, dict):
317
+ data = inline.get("data")
318
+ if not data:
319
+ return None
320
+ return hashlib.sha1(str(data).encode("ascii", errors="replace")).hexdigest()
321
+
322
+
323
+ def _has_function_response(content: Any) -> bool:
324
+ return isinstance(content, dict) and any(
325
+ isinstance(part, dict) and ("functionResponse" in part or "function_response" in part)
326
+ for part in content.get("parts") or [])
327
+
328
+
329
+ def replaceable_tool_attachment(body: dict, path: list[str | int]) -> bool:
330
+ """An attachment a tool returned, at a place where a text note is valid:
331
+ Anthropic `messages[i].content[j].content[k]` (inside a tool_result); Gemini
332
+ `contents[i].parts[j]` next to a functionResponse (Gemini CLI's sibling parts). OpenAI chat
333
+ tool messages carry text only, so tool images arrive as user messages: never changed."""
334
+ if len(path) == 6 and path[0] == "messages" and path[2] == "content" and path[4] == "content":
335
+ return True
336
+ if len(path) == 4 and path[0] == "contents" and path[2] == "parts":
337
+ return _has_function_response(body["contents"][path[1]])
338
+ return False
339
+
340
+
341
+ def note_block(path: list[str | int], note: str) -> dict:
342
+ """A text block in the provider's own format."""
343
+ return {"text": note} if path[0] == "contents" else {"type": "text", "text": note}
344
+
345
+
346
+ def duplicate_note(number: int) -> str:
347
+ return (f"[TokenCut: identical to attachment #{number} earlier in this conversation "
348
+ "(same bytes), not sent again]")
349
+
350
+
351
+ # ---------- step 5: shrink large images where the provider bills by size ----------
352
+
353
+ TARGET_LONG_EDGE = 1568 # what standard-tier Claude models always receive
354
+ MIN_SAVING = 0.10 # resize only if the provider's own formula saves at least this share
355
+ _RESIZE_FORMATS = {"PNG", "JPEG", "WEBP"}
356
+ _KEEP_DETAIL = re.compile(
357
+ r"\bzoom|\bpixel|\btiny\b|\bsmall (?:text|font|print)|\bfine print|\bread (?:the )?exact",
358
+ re.IGNORECASE)
359
+ _REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
360
+ _RESIZE_CACHE_MAX = 32
361
+ _resize_cache: dict[tuple, str | None] = {} # base64 of the cropped / resized image
362
+ _box_cache: dict[tuple[int, str, str], tuple[int, int, int, int] | None] = {}
363
+
364
+
365
+ def tool_attachment(body: dict, path: list[str | int]) -> bool:
366
+ """An attachment a tool returned (any provider format TokenCut can recognise)."""
367
+ if replaceable_tool_attachment(body, path):
368
+ return True
369
+ # Gemini CLI on multimodal models: contents[i].parts[j].functionResponse.parts[m]
370
+ return len(path) == 7 and path[0] == "contents" and path[5] == "parts"
371
+
372
+
373
+ def _tool_name(body: dict, path: list[str | int]) -> str:
374
+ if path[0] == "messages":
375
+ call_id = body["messages"][path[1]]["content"][path[3]].get("tool_use_id")
376
+ for message in body["messages"][:path[1]]:
377
+ for block in message.get("content") or [] if isinstance(message, dict) else []:
378
+ if isinstance(block, dict) and block.get("id") == call_id:
379
+ return str(block.get("name", ""))
380
+ return ""
381
+ for part in body["contents"][path[1]].get("parts") or []:
382
+ response = part.get("functionResponse") or part.get("function_response") if isinstance(
383
+ part, dict) else None
384
+ if isinstance(response, dict):
385
+ return str(response.get("name", ""))
386
+ return ""
387
+
388
+
389
+ def _computer_use(body: dict) -> bool:
390
+ return any(isinstance(t, dict) and (str(t.get("type", "")).startswith("computer")
391
+ or "computer" in str(t.get("name", "")).lower())
392
+ for t in body.get("tools") or [])
393
+
394
+
395
+ def attachment_questions(segments: list[Segment]) -> dict[str, str]:
396
+ """Per attachment segment: the user's last message before it plus the user text sent
397
+ with it (same message). Never a later message, so every decision based on it is the same
398
+ on every turn (prompt cache). Only role "user": agents such as Claude Code send their own
399
+ context (environment, skills list) as role "system" messages inside the conversation."""
400
+ with_message: dict[int, list[str]] = {}
401
+ for seg in segments:
402
+ if seg.kind == SegmentKind.USER_TEXT and seg.role == "user":
403
+ text = _REMINDER.sub("", seg.text).strip()
404
+ if text:
405
+ with_message.setdefault(seg.message_index, []).append(text)
406
+ questions, last = {}, ""
407
+ for seg in segments:
408
+ if seg.kind == SegmentKind.USER_TEXT and seg.role == "user":
409
+ text = _REMINDER.sub("", seg.text).strip()
410
+ if text:
411
+ last = text
412
+ elif seg.kind == SegmentKind.MEDIA:
413
+ questions[seg.id] = "\n".join([last, *with_message.get(seg.message_index, [])])
414
+ return questions
415
+
416
+
417
+ def _image_data(block: dict) -> tuple[str, str] | None:
418
+ """(mime, base64 data) of an inline image block in any provider format."""
419
+ if block.get("type") == "image":
420
+ source = block.get("source") or {}
421
+ if source.get("type") == "base64":
422
+ return str(source.get("media_type", "")), str(source.get("data", ""))
423
+ return None
424
+ if block.get("type") == "image_url":
425
+ return _data_url(str((block.get("image_url") or {}).get("url", "")))
426
+ inline = block.get("inlineData") or block.get("inline_data")
427
+ if isinstance(inline, dict):
428
+ return str(inline.get("mimeType") or inline.get("mime_type") or ""), str(inline.get("data", ""))
429
+ return None
430
+
431
+
432
+ def _with_image_data(block: dict, data: str) -> dict:
433
+ new = copy.deepcopy(block)
434
+ if new.get("type") == "image":
435
+ new["source"]["data"] = data
436
+ elif new.get("type") == "image_url":
437
+ url = new["image_url"]["url"]
438
+ new["image_url"]["url"] = url[:url.index(",") + 1] + data
439
+ else:
440
+ key = "inlineData" if "inlineData" in new else "inline_data"
441
+ new[key]["data"] = data
442
+ return new
443
+
444
+
445
+ def _image_tokens(provider: str, model: str, w: int, h: int, block: dict, body: dict) -> int:
446
+ provider = pricing_provider(provider, model)
447
+ if provider == "anthropic":
448
+ return claude_image_tokens(w, h, model)
449
+ if provider == "openai":
450
+ return openai_image_tokens(w, h, model, str((block.get("image_url") or {}).get("detail", "auto")))
451
+ config = body.get("generationConfig") or body.get("generation_config") or {}
452
+ return gemini_image_tokens(w, h, model, str(config.get("mediaResolution")
453
+ or config.get("media_resolution") or ""))
454
+
455
+
456
+ def _adjusted_b64(data: str, box: tuple[int, int, int, int] | None,
457
+ size: tuple[int, int] | None) -> str | None:
458
+ """Crop to `box`, then resize to `size`. Same input -> same output bytes (fixed filter and
459
+ settings), cached per image and operation."""
460
+ key = (len(data), data[:2048], data[-2048:], box, size)
461
+ if key not in _resize_cache:
462
+ if len(_resize_cache) >= _RESIZE_CACHE_MAX:
463
+ _resize_cache.pop(next(iter(_resize_cache)))
464
+ raw = _transform(_decode(data), box, size)
465
+ _resize_cache[key] = base64.b64encode(raw).decode() if raw is not None else None
466
+ return _resize_cache[key]
467
+
468
+
469
+ def _open_image(raw: bytes) -> Image.Image | None:
470
+ try:
471
+ img = Image.open(io.BytesIO(raw))
472
+ if img.format not in _RESIZE_FORMATS or getattr(img, "is_animated", False):
473
+ return None
474
+ img.load()
475
+ return img
476
+ except (OSError, ValueError, Image.DecompressionBombError):
477
+ return None
478
+
479
+
480
+ def _transform(raw: bytes, box: tuple[int, int, int, int] | None,
481
+ size: tuple[int, int] | None) -> bytes | None:
482
+ img = _open_image(raw)
483
+ if img is None:
484
+ return None
485
+ fmt = img.format
486
+ out = img.crop(box) if box is not None else img
487
+ if size is not None and size != out.size:
488
+ out = out.resize(size, Image.Resampling.LANCZOS)
489
+ buf = io.BytesIO()
490
+ if fmt == "JPEG":
491
+ out.convert("RGB").save(buf, "JPEG", quality=90)
492
+ elif fmt == "WEBP":
493
+ out.save(buf, "WEBP", quality=90)
494
+ else:
495
+ out.save(buf, "PNG")
496
+ return buf.getvalue()
497
+
498
+
499
+ # ---------- step 6: plain outer borders are cut off ----------
500
+
501
+ BORDER_TOLERANCE = 16 # max per-channel difference still counted as background
502
+ BORDER_PAD = 8 # px of the border kept around the content
503
+ BORDER_MAX_REMOVED = 0.75 # cutting more than this share: a near-empty image, left alone
504
+ BORDER_MIN_CONTENT = 32 # px; smaller content boxes are left alone
505
+ _LAYOUT_QUESTION = re.compile(
506
+ r"\b(?:layout|align\w*|spacing|spaced|margins?|padding|cent(?:er|re)d?|position\w*|"
507
+ r"whitespace|white space|gaps?|offset|overflow\w*|cut off|borders?)\b", re.IGNORECASE)
508
+
509
+
510
+ def content_box(img: Image.Image) -> tuple[int, int, int, int] | None:
511
+ """Box (left, top, right, bottom) around everything that differs from a plain background,
512
+ plus BORDER_PAD px; None if the corners do not share one colour or nothing differs."""
513
+ rgb = img.convert("RGB")
514
+ w, h = rgb.size
515
+ corners = [rgb.getpixel(p) for p in ((0, 0), (w - 1, 0), (0, h - 1), (w - 1, h - 1))]
516
+ background = corners[0]
517
+ if any(max(abs(a - b) for a, b in zip(c, background, strict=True)) > BORDER_TOLERANCE
518
+ for c in corners):
519
+ return None
520
+ red, green, blue = ImageChops.difference(rgb, Image.new("RGB", rgb.size, background)).split()
521
+ strongest = ImageChops.lighter(ImageChops.lighter(red, green), blue)
522
+ box = strongest.point(lambda v: 255 if v > BORDER_TOLERANCE else 0).getbbox()
523
+ if box is None:
524
+ return None
525
+ left, top, right, bottom = box
526
+ return (max(0, left - BORDER_PAD), max(0, top - BORDER_PAD),
527
+ min(w, right + BORDER_PAD), min(h, bottom + BORDER_PAD))
528
+
529
+
530
+ def _border_box(data: str, w: int, h: int) -> tuple[int, int, int, int] | None:
531
+ """The content box worth cropping to, cached per image."""
532
+ key = (len(data), data[:2048], data[-2048:])
533
+ if key not in _box_cache:
534
+ if len(_box_cache) >= _RESIZE_CACHE_MAX:
535
+ _box_cache.pop(next(iter(_box_cache)))
536
+ img = _open_image(_decode(data))
537
+ box = content_box(img) if img is not None else None
538
+ if box is not None:
539
+ bw, bh = box[2] - box[0], box[3] - box[1]
540
+ if (bw, bh) == (w, h) or min(bw, bh) < BORDER_MIN_CONTENT \
541
+ or 1 - (bw * bh) / (w * h) > BORDER_MAX_REMOVED:
542
+ box = None
543
+ _box_cache[key] = box
544
+ return _box_cache[key]
545
+
546
+
547
+ def _image_note(w: int, h: int, box: tuple[int, int, int, int], size: tuple[int, int]) -> str:
548
+ bw, bh = box[2] - box[0], box[3] - box[1]
549
+ scaled = f", then scaled to {size[0]}x{size[1]}" if size != (bw, bh) else ""
550
+ return (f"[TokenCut: plain border cut from this image: original {w}x{h}, kept the "
551
+ f"{bw}x{bh} area at offset ({box[0]}, {box[1]}){scaled}; positions refer to the "
552
+ "image as sent]")
553
+
554
+
555
+ def _scaled_note(w: int, h: int, size: tuple[int, int]) -> str:
556
+ return (f"[TokenCut: image scaled from {w}x{h} to {size[0]}x{size[1]} to save tokens; "
557
+ "positions refer to the image as sent]")
558
+
559
+
560
+ def adjust_images(provider: str, model: str, body: dict, segments: list[Segment],
561
+ include_user: bool = False, resize: bool = True, trim: bool = True
562
+ ) -> tuple[list[tuple[list[str | int], list[dict]]], list[dict]]:
563
+ """(path -> replacement blocks, report entries): plain borders cut (step 6) and large images
564
+ shrunk (step 5), only where the provider's own formula saves >= MIN_SAVING. The body is not
565
+ changed; a cropped image is followed by a note, so insert in descending path order."""
566
+ if _computer_use(body) or not (resize or trim):
567
+ return [], []
568
+ questions = attachment_questions(segments)
569
+ changes, entries = [], []
570
+ for seg in segments:
571
+ if seg.kind != SegmentKind.MEDIA:
572
+ continue
573
+ block = lookup(body, seg.path)
574
+ image = _image_data(block) if isinstance(block, dict) else None
575
+ if image is None or not image[0].startswith("image/"):
576
+ continue
577
+ is_tool = tool_attachment(body, seg.path)
578
+ if not (is_tool or include_user):
579
+ continue
580
+ if is_tool and "computer" in _tool_name(body, seg.path).lower():
581
+ continue
582
+ size = image_size(_decode(image[1], HEAD_B64_CHARS))
583
+ if size is None:
584
+ continue
585
+ w, h = size
586
+ # a note can follow the image except inside Gemini's functionResponse.parts
587
+ can_note = not (len(seg.path) == 7 and seg.path[0] == "contents")
588
+ question = questions.get(seg.id, "")
589
+ trim_here = trim and can_note and not _LAYOUT_QUESTION.search(question)
590
+ resize_here = resize and not _KEEP_DETAIL.search(question)
591
+ box = _border_box(image[1], w, h) if trim_here else None
592
+ cw, ch = (box[2] - box[0], box[3] - box[1]) if box is not None else (w, h)
593
+ target = (cw, ch)
594
+ if resize_here and max(cw, ch) > TARGET_LONG_EDGE:
595
+ scale = TARGET_LONG_EDGE / max(cw, ch)
596
+ target = (max(1, round(cw * scale)), max(1, round(ch * scale)))
597
+ if box is None and target == (w, h):
598
+ continue
599
+ before = _image_tokens(provider, model, w, h, block, body)
600
+ after = _image_tokens(provider, model, *target, block, body)
601
+ if before - after < MIN_SAVING * before:
602
+ continue
603
+ encoded = _adjusted_b64(image[1], box, target)
604
+ if encoded is None:
605
+ continue
606
+ blocks = [_with_image_data(block, encoded)]
607
+ if box is not None:
608
+ blocks.append(note_block(seg.path, _image_note(w, h, box, target)))
609
+ elif can_note:
610
+ blocks.append(note_block(seg.path, _scaled_note(w, h, target)))
611
+ how = ("border_trimmed_resized" if target != (cw, ch) else "border_trimmed") \
612
+ if box is not None else "resized"
613
+ changes.append((seg.path, blocks))
614
+ entries.append({"segment": seg.id, "how": how, "tool": "", "kind": "image",
615
+ "from_size": [w, h], "to_size": list(target),
616
+ "crop_box": list(box) if box is not None else None,
617
+ "before": before,
618
+ "after": after})
619
+ return changes, entries
620
+
621
+
622
+ # ---------- step 7: PDF pages without images or drawings are sent as text ----------
623
+
624
+ PDF_MIN_TEXT_CHARS = 200 # less text than this: probably a scan or a figure page
625
+ PDF_MAX_DRAW_OPS = 20 # more line / box / curve operators: probably a table or a chart
626
+ PDF_TABLE_ROWS = 3 # this many number-table rows: a table (columns are lost as plain text)
627
+ PDF_MAX_PAGES = 50 # longer PDFs are left as they are (~115 ms per page the first time)
628
+ _DRAW_OP = re.compile(rb"(?<![A-Za-z])(?:re|l|c|v|y)(?=\s)")
629
+ _NUMBER_CELL = re.compile(r"[-+]?\d+(?:[.,]\d+)*%?")
630
+ # Intent safety: a question about how pages look keeps the whole PDF (EcoDoc, ACL 2025, also
631
+ # chooses text or image per page from page characteristics and query intent).
632
+ _VISUAL_QUESTION = re.compile(
633
+ r"\b(?:charts?|graphs?|plots?|figures?|fig\.|diagrams?|images?|pictures?|photos?|"
634
+ r"logos?|signatures?|signed|stamps?|seals?|handwrit\w*|colou?rs?|layout|formatting|"
635
+ r"(?<!answer )formats?(?!:|\s+(?:your|the|this)\s+(?:final\s+)?(?:answer|reply|response))|"
636
+ r"fonts?|scan(?:ned|s)?|tables?|maps?|drawings?|screenshots?|visual\w*|"
637
+ r"looks? like|appearance|highlight\w*)", re.IGNORECASE)
638
+ _PDF_ERRORS = (PyPdfError, ValueError, KeyError, TypeError, AttributeError, OSError)
639
+ _SPLIT_CACHE_MAX = 32
640
+ _split_cache: dict[tuple[int, str, str], tuple[str, str | None, list[int], int] | None] = {}
641
+
642
+
643
+ def _draw_ops(page: Any) -> int:
644
+ """Line / box / curve operators on the page and in its form XObjects (vector charts)."""
645
+ contents = page.get_contents()
646
+ total = len(_DRAW_OP.findall(contents.get_data())) if contents is not None else 0
647
+ resources = page.get("/Resources")
648
+ xobjects = resources.get_object().get("/XObject") if resources is not None else None
649
+ for ref in (xobjects.get_object().values() if xobjects is not None else []):
650
+ obj = ref.get_object()
651
+ if obj.get("/Subtype") == "/Form":
652
+ total += len(_DRAW_OP.findall(obj.get_data()))
653
+ return total
654
+
655
+
656
+ def _number_table_rows(page: Any) -> int:
657
+ """Rows with >= 4 columns of which >= 3 are numbers, in layout-preserving text."""
658
+ rows = 0
659
+ for line in (page.extract_text(extraction_mode="layout") or "").splitlines():
660
+ cells = [c for c in re.split(r" {2,}", line.strip()) if c]
661
+ if len(cells) >= 4 and sum(1 for c in cells if _NUMBER_CELL.fullmatch(c)) >= 3:
662
+ rows += 1
663
+ return rows
664
+
665
+
666
+ def pdf_page_kind(page: Any) -> tuple[str, str]:
667
+ """("text" / "visual", page text). A page is "text" only with enough text, no raster
668
+ images, almost no drawings and no number-table rows; scans, figures, charts and tables
669
+ stay "visual". Raises one of _PDF_ERRORS on an unreadable page."""
670
+ text = (page.extract_text() or "").strip()
671
+ visual = (len(page.images) > 0 or len(text) < PDF_MIN_TEXT_CHARS
672
+ or _draw_ops(page) > PDF_MAX_DRAW_OPS
673
+ or _number_table_rows(page) >= PDF_TABLE_ROWS)
674
+ return ("visual" if visual else "text"), text
675
+
676
+
677
+ def _analyse_pdf(raw: bytes) -> tuple[Any, list[str], list[str]] | None:
678
+ """(reader, per-page "text" / "visual", per-page text), or None if unreadable / too long."""
679
+ try:
680
+ reader = PdfReader(io.BytesIO(raw))
681
+ if reader.is_encrypted or len(reader.pages) > PDF_MAX_PAGES:
682
+ return None
683
+ kinds, texts = [], []
684
+ for page in reader.pages:
685
+ kind, text = pdf_page_kind(page)
686
+ kinds.append(kind)
687
+ texts.append(text)
688
+ return reader, kinds, texts
689
+ except _PDF_ERRORS:
690
+ return None
691
+
692
+
693
+ def pdf_page_kinds(raw: bytes) -> list[str] | None:
694
+ analysed = _analyse_pdf(raw)
695
+ return analysed[1] if analysed else None
696
+
697
+
698
+ def _split_pdf(data: str) -> tuple[str, str | None, list[int], int] | None:
699
+ """(text of the text pages, base64 PDF of the other pages or None, text page numbers,
700
+ page count), or None if there is nothing to send as text. Same bytes every time."""
701
+ key = (len(data), data[:2048], data[-2048:])
702
+ if key in _split_cache:
703
+ return _split_cache[key]
704
+ result = None
705
+ analysed = _analyse_pdf(_decode(data))
706
+ if analysed and "text" in analysed[1]:
707
+ reader, kinds, texts = analysed
708
+ try:
709
+ text_pages = [n for n, kind in enumerate(kinds) if kind == "text"]
710
+ text = "\n\n".join(f"--- page {n + 1} ---\n{texts[n]}" for n in text_pages)
711
+ visual = [n for n, kind in enumerate(kinds) if kind == "visual"]
712
+ subset = None
713
+ if visual:
714
+ writer = PdfWriter()
715
+ for n in visual:
716
+ writer.add_page(reader.pages[n])
717
+ buf = io.BytesIO()
718
+ writer.write(buf)
719
+ subset = base64.b64encode(buf.getvalue()).decode()
720
+ result = (text, subset, [n + 1 for n in text_pages], len(kinds))
721
+ except _PDF_ERRORS:
722
+ result = None
723
+ if len(_split_cache) >= _SPLIT_CACHE_MAX:
724
+ _split_cache.pop(next(iter(_split_cache)))
725
+ _split_cache[key] = result
726
+ return result
727
+
728
+
729
+ def _read_pdf_tool(body: dict) -> str:
730
+ """Name of the agent's read_pdf tool (tokencut/pdftool), or "". The tool list starts the
731
+ cached prefix, so a note based on it never changes an earlier turn on its own."""
732
+ for tool in body.get("tools") or []:
733
+ if isinstance(tool, dict):
734
+ name = str(tool.get("name") or (tool.get("function") or {}).get("name") or "")
735
+ if name.endswith("read_pdf"):
736
+ return name
737
+ return ""
738
+
739
+
740
+ def _pdf_note(name: str, text_pages: list[int], total: int, visual_left: bool,
741
+ reader: str = "") -> str:
742
+ pages = ", ".join(str(n) for n in text_pages)
743
+ rest = "the other pages follow as PDF" if visual_left else "no page images sent"
744
+ label = f" '{name}'" if name else ""
745
+ ask = (f"; to see one of these pages as a picture, call {reader} with as_images=true"
746
+ if reader else "")
747
+ return (f"[TokenCut: PDF{label}, pages {pages} of {total} sent as text (no images or "
748
+ f"drawings found on them); {rest}{ask}]")
749
+
750
+
751
+ def split_pdfs(provider: str, body: dict, segments: list[Segment], include_user: bool = False
752
+ ) -> tuple[list[tuple[list[str | int], list[dict]]], list[dict]]:
753
+ """(path -> replacement blocks, report entries) for PDFs with text pages (Claude, OpenAI).
754
+ Insert the replacements in descending path order: one block may become two."""
755
+ if provider not in ("anthropic", "openai"):
756
+ return [], [] # Gemini: its docs conflict on whether PDF text is billed
757
+ changes, entries = [], []
758
+ questions = attachment_questions(segments)
759
+ reader = _read_pdf_tool(body)
760
+ for seg in segments:
761
+ if seg.kind != SegmentKind.MEDIA:
762
+ continue
763
+ block = lookup(body, seg.path)
764
+ if not isinstance(block, dict):
765
+ continue
766
+ if not (tool_attachment(body, seg.path) or include_user):
767
+ continue
768
+ if _VISUAL_QUESTION.search(questions.get(seg.id, "")):
769
+ continue # the question is about how pages look: keep every page picture
770
+ if provider == "anthropic" and block.get("type") == "document":
771
+ source = block.get("source") or {}
772
+ if source.get("type") != "base64" or source.get("media_type") != "application/pdf":
773
+ continue
774
+ data, name = str(source.get("data", "")), str(block.get("title", ""))
775
+ elif provider == "openai" and block.get("type") == "file":
776
+ parsed = _data_url(str((block.get("file") or {}).get("file_data", "")))
777
+ if parsed is None or parsed[0] != "application/pdf":
778
+ continue
779
+ data, name = parsed[1], str((block.get("file") or {}).get("filename", ""))
780
+ else:
781
+ continue
782
+ split = _split_pdf(data)
783
+ if split is None:
784
+ continue
785
+ text, subset, text_pages, total = split
786
+ note = _pdf_note(name, text_pages, total, subset is not None, reader)
787
+ blocks = [{"type": "text", "text": f"{note}\n\n{text}"}]
788
+ if subset is not None:
789
+ rest = copy.deepcopy(block)
790
+ if provider == "anthropic":
791
+ rest["source"]["data"] = subset
792
+ else:
793
+ rest["file"]["file_data"] = "data:application/pdf;base64," + subset
794
+ blocks.append(rest)
795
+ elif "cache_control" in block:
796
+ blocks[0]["cache_control"] = block["cache_control"]
797
+ changes.append((seg.path, blocks))
798
+ entries.append({"segment": seg.id, "how": "pdf_text_pages", "tool": "",
799
+ "before": seg.token_count, "after": seg.token_count, "pages": total,
800
+ "page_images_removed": len(text_pages), "kind": "pdf",
801
+ "text_pages": text_pages,
802
+ "visual_pages": [n for n in range(1, total + 1) if n not in text_pages]})
803
+ return changes, entries
804
+
805
+
806
+ def with_media_tokens(provider: str, model: str, body: dict,
807
+ segments: list[Segment]) -> tuple[list[Segment], dict]:
808
+ """Segments with estimated attachment tokens, and counts for the analysis."""
809
+ stats = {"images": 0, "documents": 0, "unknown": 0, "tokens": 0}
810
+ out = []
811
+ for seg in segments:
812
+ found = None
813
+ if seg.kind == SegmentKind.MEDIA:
814
+ found = _estimate(provider, model, lookup(body, seg.path), body)
815
+ if found is None:
816
+ out.append(seg)
817
+ continue
818
+ kind, tokens = found
819
+ stats["images" if kind == "image" else "documents"] += 1
820
+ if tokens is None:
821
+ stats["unknown"] += 1
822
+ tokens = 0
823
+ stats["tokens"] += tokens
824
+ out.append(seg.model_copy(update={"token_count": tokens}))
825
+ return out, stats