tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,825 @@
|
|
|
1
|
+
"""Attachment token estimates: images and documents (docs/ATTACHMENTS_PLAN.md, step 1).
|
|
2
|
+
|
|
3
|
+
Attachments are forwarded unchanged (`SegmentKind.MEDIA`). Their text is empty, so until now
|
|
4
|
+
they counted as 0 tokens. This module estimates what the provider bills for them, from the
|
|
5
|
+
formulas the providers publish (read Nov 2026, see the plan's sources), so stats and later
|
|
6
|
+
cost decisions see them. Estimates are in the provider's own tokens, not proxy tokens.
|
|
7
|
+
|
|
8
|
+
Only headers are read (no image library). Unknown sizes (an image behind a URL or a file id)
|
|
9
|
+
count as 0 and are reported as `unknown`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import base64
|
|
13
|
+
import binascii
|
|
14
|
+
import copy
|
|
15
|
+
import hashlib
|
|
16
|
+
import io
|
|
17
|
+
import math
|
|
18
|
+
import re
|
|
19
|
+
import struct
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from PIL import Image, ImageChops
|
|
23
|
+
from pypdf import PdfReader, PdfWriter
|
|
24
|
+
from pypdf.errors import PyPdfError
|
|
25
|
+
|
|
26
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
27
|
+
from tokencut.gateway.adapters.common import lookup
|
|
28
|
+
from tokencut.gateway.routing import DEFAULT_GPT_PREFIXES
|
|
29
|
+
from tokencut.tokens import count_tokens
|
|
30
|
+
|
|
31
|
+
HEAD_B64_CHARS = 1 << 18 # 256k base64 characters (~192 KB): enough to reach an image header
|
|
32
|
+
PDF_TEXT_TOKENS_PER_PAGE = 2_250 # Claude: "typically 1,500-3,000" text tokens per page (midpoint)
|
|
33
|
+
_PDF_CACHE_MAX = 64
|
|
34
|
+
_pdf_cache: dict[tuple[int, str, str], int | None] = {}
|
|
35
|
+
|
|
36
|
+
# ---------- reading sizes ----------
|
|
37
|
+
|
|
38
|
+
_JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def image_size(raw: bytes) -> tuple[int, int] | None:
|
|
42
|
+
"""(width, height) from a PNG / JPEG / GIF / WebP header, or None."""
|
|
43
|
+
try:
|
|
44
|
+
if raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR":
|
|
45
|
+
return struct.unpack(">II", raw[16:24])
|
|
46
|
+
if raw[:6] in (b"GIF87a", b"GIF89a"):
|
|
47
|
+
return struct.unpack("<HH", raw[6:10])
|
|
48
|
+
if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP":
|
|
49
|
+
return _webp_size(raw)
|
|
50
|
+
if raw[:2] == b"\xff\xd8":
|
|
51
|
+
return _jpeg_size(raw)
|
|
52
|
+
except struct.error:
|
|
53
|
+
return None
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _webp_size(raw: bytes) -> tuple[int, int] | None:
|
|
58
|
+
chunk = raw[12:16]
|
|
59
|
+
if chunk == b"VP8X":
|
|
60
|
+
return 1 + int.from_bytes(raw[24:27], "little"), 1 + int.from_bytes(raw[27:30], "little")
|
|
61
|
+
if chunk == b"VP8L":
|
|
62
|
+
bits = int.from_bytes(raw[21:25], "little")
|
|
63
|
+
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
|
64
|
+
if chunk == b"VP8 ":
|
|
65
|
+
w, h = struct.unpack("<HH", raw[26:30])
|
|
66
|
+
return w & 0x3FFF, h & 0x3FFF
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _jpeg_size(raw: bytes) -> tuple[int, int] | None:
|
|
71
|
+
i = 2
|
|
72
|
+
while i + 9 <= len(raw):
|
|
73
|
+
if raw[i] != 0xFF:
|
|
74
|
+
i += 1
|
|
75
|
+
continue
|
|
76
|
+
marker = raw[i + 1]
|
|
77
|
+
if marker in _JPEG_SOF:
|
|
78
|
+
h, w = struct.unpack(">HH", raw[i + 5:i + 9])
|
|
79
|
+
return w, h
|
|
80
|
+
if marker == 0xFF: # fill byte
|
|
81
|
+
i += 1
|
|
82
|
+
elif marker in (0x01, 0xD8) or 0xD0 <= marker <= 0xD7: # markers without a length
|
|
83
|
+
i += 2
|
|
84
|
+
else:
|
|
85
|
+
i += 2 + struct.unpack(">H", raw[i + 2:i + 4])[0]
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def pdf_pages(raw: bytes) -> int | None:
|
|
90
|
+
"""Page count from the page objects, else from the page tree's /Count, else (page objects
|
|
91
|
+
inside compressed object streams, PDF 1.5+) from the PDF parser."""
|
|
92
|
+
if not raw.startswith(b"%PDF"):
|
|
93
|
+
return None
|
|
94
|
+
pages = len(re.findall(rb"/Type\s*/Page(?![A-Za-z])", raw))
|
|
95
|
+
if pages:
|
|
96
|
+
return pages
|
|
97
|
+
counts = [int(c) for c in re.findall(rb"/Type\s*/Pages\b[^>]*?/Count\s+(\d+)", raw)]
|
|
98
|
+
if counts:
|
|
99
|
+
return max(counts)
|
|
100
|
+
try:
|
|
101
|
+
return len(PdfReader(io.BytesIO(raw)).pages) or None
|
|
102
|
+
except _PDF_ERRORS:
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _decode(data: str, limit: int | None = None) -> bytes:
|
|
107
|
+
part = data if limit is None else data[:limit - limit % 4]
|
|
108
|
+
try:
|
|
109
|
+
return base64.b64decode(part, validate=False)
|
|
110
|
+
except (binascii.Error, ValueError):
|
|
111
|
+
return b""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _pdf_pages_b64(data: str) -> int | None:
|
|
115
|
+
"""Whole document decoded once per distinct PDF (the same PDF is re-sent every turn)."""
|
|
116
|
+
key = (len(data), data[:2048], data[-2048:])
|
|
117
|
+
if key not in _pdf_cache:
|
|
118
|
+
if len(_pdf_cache) >= _PDF_CACHE_MAX:
|
|
119
|
+
_pdf_cache.pop(next(iter(_pdf_cache)))
|
|
120
|
+
_pdf_cache[key] = pdf_pages(_decode(data))
|
|
121
|
+
return _pdf_cache[key]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------- provider formulas ----------
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def claude_high_res(model: str) -> bool:
|
|
128
|
+
"""Claude 4.7 and later use the high-resolution image tier."""
|
|
129
|
+
m = re.search(r"claude-(?:[a-z]+-)?(\d+)-(\d{1,2})(?!\d)", model) or re.search(
|
|
130
|
+
r"claude-(?:[a-z]+-)?(\d+)", model)
|
|
131
|
+
if not m:
|
|
132
|
+
return False
|
|
133
|
+
major = int(m.group(1))
|
|
134
|
+
minor = int(m.group(2)) if m.lastindex and m.lastindex >= 2 else 0
|
|
135
|
+
return (major, minor) >= (4, 7)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _claude_tokens(w: int, h: int) -> int:
|
|
139
|
+
return math.ceil(w / 28) * math.ceil(h / 28)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def claude_image_tokens(w: int, h: int, model: str) -> int:
|
|
143
|
+
"""ceil(w/28) x ceil(h/28); larger images are shrunk (aspect kept) to the tier's limits."""
|
|
144
|
+
edge, cap = (2576, 4784) if claude_high_res(model) else (1568, 1568)
|
|
145
|
+
if max(w, h) <= edge and _claude_tokens(w, h) <= cap:
|
|
146
|
+
return _claude_tokens(w, h)
|
|
147
|
+
long_side, short_side = max(w, h), min(w, h)
|
|
148
|
+
side = min(edge, long_side)
|
|
149
|
+
while side > 1:
|
|
150
|
+
other = max(1, round(side * short_side / long_side))
|
|
151
|
+
if _claude_tokens(side, other) <= cap:
|
|
152
|
+
return _claude_tokens(side, other)
|
|
153
|
+
side -= 1
|
|
154
|
+
return 1
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# (base, per tile) for tile-priced OpenAI models; first matching prefix wins
|
|
158
|
+
_OPENAI_TILES = (("gpt-4o-mini", (2833, 5667)), ("gpt-4.1-mini", None), ("gpt-4.1-nano", None),
|
|
159
|
+
("gpt-4o", (85, 170)), ("gpt-4.1", (85, 170)), ("gpt-5.1", (70, 140)))
|
|
160
|
+
_OPENAI_PATCH_MULTIPLIER = (("gpt-4.1-mini", 1.62), ("gpt-5-nano", 1.5))
|
|
161
|
+
_OPENAI_DEFAULT_MULTIPLIER = 1.2 # gpt-5.2 / 5.4 / 5.5 in the guide
|
|
162
|
+
_OPENAI_MAX_SIDE, _OPENAI_MAX_PATCHES = 2048, 2500
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _fit(w: int, h: int, box: int) -> tuple[int, int]:
|
|
166
|
+
scale = min(1.0, box / max(w, h))
|
|
167
|
+
return max(1, round(w * scale)), max(1, round(h * scale))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def openai_image_tokens(w: int, h: int, model: str, detail: str = "auto") -> int:
|
|
171
|
+
"""Tile models: base + per-tile x 512-px tiles. Patch models: 32-px patches x multiplier."""
|
|
172
|
+
name = model.lower()
|
|
173
|
+
tiles = next((t for prefix, t in _OPENAI_TILES if name.startswith(prefix)), None)
|
|
174
|
+
if tiles is not None:
|
|
175
|
+
base, per_tile = tiles
|
|
176
|
+
if detail == "low":
|
|
177
|
+
return base
|
|
178
|
+
w, h = _fit(w, h, 2048)
|
|
179
|
+
if min(w, h) > 768:
|
|
180
|
+
scale = 768 / min(w, h)
|
|
181
|
+
w, h = round(w * scale), round(h * scale)
|
|
182
|
+
return base + per_tile * math.ceil(w / 512) * math.ceil(h / 512)
|
|
183
|
+
mult = next((m for prefix, m in _OPENAI_PATCH_MULTIPLIER if name.startswith(prefix)),
|
|
184
|
+
_OPENAI_DEFAULT_MULTIPLIER)
|
|
185
|
+
if detail == "low":
|
|
186
|
+
w, h = _fit(w, h, 512)
|
|
187
|
+
elif detail != "original":
|
|
188
|
+
w, h = _fit(w, h, _OPENAI_MAX_SIDE)
|
|
189
|
+
while math.ceil(w / 32) * math.ceil(h / 32) > _OPENAI_MAX_PATCHES:
|
|
190
|
+
w, h = _fit(w, h, max(w, h) - 1)
|
|
191
|
+
return math.ceil(math.ceil(w / 32) * math.ceil(h / 32) * mult)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
_GEMINI_IMAGE = {"LOW": 280, "MEDIUM": 560, "HIGH": 1120, "ULTRA_HIGH": 2240}
|
|
195
|
+
_GEMINI_PDF_PAGE = {"LOW": 280, "MEDIUM": 560, "HIGH": 1120}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _gemini_new(model: str) -> bool:
|
|
199
|
+
m = re.search(r"gemini-(\d+)", model)
|
|
200
|
+
return m is None or int(m.group(1)) >= 3 # unknown: assume a current model
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _resolution(value: str) -> str:
|
|
204
|
+
return value.upper().removeprefix("MEDIA_RESOLUTION_")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def gemini_image_tokens(w: int, h: int, model: str, resolution: str = "") -> int:
|
|
208
|
+
if _gemini_new(model):
|
|
209
|
+
return _GEMINI_IMAGE.get(_resolution(resolution), 1120)
|
|
210
|
+
if w <= 384 and h <= 384:
|
|
211
|
+
return 258
|
|
212
|
+
unit = max(1, math.floor(min(w, h) / 1.5))
|
|
213
|
+
return 258 * math.ceil(w / unit) * math.ceil(h / unit)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def gemini_pdf_page_tokens(model: str, resolution: str = "") -> int:
|
|
217
|
+
if _gemini_new(model):
|
|
218
|
+
return _GEMINI_PDF_PAGE.get(_resolution(resolution), 560)
|
|
219
|
+
return 258
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ---------- attachments inside requests ----------
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _data_url(url: str) -> tuple[str, str] | None:
|
|
226
|
+
m = re.match(r"data:([^;,]+)[^,]*;base64,", url)
|
|
227
|
+
return (m.group(1), url[m.end():]) if m else None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _estimate(provider: str, model: str, block: Any, body: dict) -> tuple[str, int | None] | None:
|
|
231
|
+
"""(kind "image"/"document", tokens or None if unknown) for an attachment block, else None."""
|
|
232
|
+
if not isinstance(block, dict):
|
|
233
|
+
return None
|
|
234
|
+
if provider == "anthropic" and block.get("type") in ("image", "document"):
|
|
235
|
+
kind = block["type"]
|
|
236
|
+
source = block.get("source") or {}
|
|
237
|
+
if source.get("type") == "text":
|
|
238
|
+
return kind, count_tokens(str(source.get("data", "")))
|
|
239
|
+
if source.get("type") != "base64":
|
|
240
|
+
return kind, None
|
|
241
|
+
return kind, _from_bytes(provider, model, str(source.get("media_type", "")),
|
|
242
|
+
str(source.get("data", "")), "")
|
|
243
|
+
if provider == "openai":
|
|
244
|
+
if block.get("type") == "image_url":
|
|
245
|
+
image = block.get("image_url") or {}
|
|
246
|
+
parsed = _data_url(str(image.get("url", "")))
|
|
247
|
+
if parsed is None:
|
|
248
|
+
return "image", None
|
|
249
|
+
return "image", _from_bytes(provider, model, parsed[0], parsed[1],
|
|
250
|
+
str(image.get("detail", "auto")))
|
|
251
|
+
if block.get("type") == "file":
|
|
252
|
+
parsed = _data_url(str((block.get("file") or {}).get("file_data", "")))
|
|
253
|
+
if parsed is None:
|
|
254
|
+
return "document", None
|
|
255
|
+
return "document", _from_bytes(provider, model, parsed[0], parsed[1], "")
|
|
256
|
+
return None
|
|
257
|
+
if provider == "gemini":
|
|
258
|
+
inline = block.get("inlineData") or block.get("inline_data")
|
|
259
|
+
file_data = block.get("fileData") or block.get("file_data")
|
|
260
|
+
if isinstance(file_data, dict):
|
|
261
|
+
mime = str(file_data.get("mimeType") or file_data.get("mime_type") or "")
|
|
262
|
+
return ("image" if mime.startswith("image/") else "document"), None
|
|
263
|
+
if not isinstance(inline, dict):
|
|
264
|
+
return None
|
|
265
|
+
mime = str(inline.get("mimeType") or inline.get("mime_type") or "")
|
|
266
|
+
config = body.get("generationConfig") or body.get("generation_config") or {}
|
|
267
|
+
resolution = str(config.get("mediaResolution") or config.get("media_resolution") or "")
|
|
268
|
+
kind = "image" if mime.startswith("image/") else "document"
|
|
269
|
+
return kind, _from_bytes(provider, model, mime, str(inline.get("data", "")), resolution)
|
|
270
|
+
return None
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def pricing_provider(provider: str, model: str) -> str:
|
|
274
|
+
"""Whose image / PDF prices apply. Claude Code sends GPT models in Claude's format when
|
|
275
|
+
TokenCut routes them through LiteLLM to the ChatGPT plan: those are priced as OpenAI."""
|
|
276
|
+
if provider == "anthropic" and model.lower().startswith(DEFAULT_GPT_PREFIXES):
|
|
277
|
+
return "openai"
|
|
278
|
+
return provider
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _from_bytes(provider: str, model: str, mime: str, data: str, option: str) -> int | None:
|
|
282
|
+
provider = pricing_provider(provider, model)
|
|
283
|
+
if mime.startswith("image/"):
|
|
284
|
+
size = image_size(_decode(data, HEAD_B64_CHARS))
|
|
285
|
+
if size is None:
|
|
286
|
+
return None
|
|
287
|
+
w, h = size
|
|
288
|
+
if provider == "anthropic":
|
|
289
|
+
return claude_image_tokens(w, h, model)
|
|
290
|
+
if provider == "openai":
|
|
291
|
+
return openai_image_tokens(w, h, model, option or "auto")
|
|
292
|
+
return gemini_image_tokens(w, h, model, option)
|
|
293
|
+
if mime == "application/pdf":
|
|
294
|
+
pages = _pdf_pages_b64(data)
|
|
295
|
+
if pages is None:
|
|
296
|
+
return None
|
|
297
|
+
if provider == "gemini":
|
|
298
|
+
return pages * gemini_pdf_page_tokens(model, option)
|
|
299
|
+
return pages * PDF_TEXT_TOKENS_PER_PAGE # text side only: page images come on top
|
|
300
|
+
if mime.startswith("text/"):
|
|
301
|
+
return count_tokens(_decode(data).decode("utf-8", errors="replace"))
|
|
302
|
+
return None
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def attachment_key(block: Any) -> str | None:
|
|
306
|
+
"""Content hash of an inline attachment (Anthropic base64 block, Gemini inlineData)."""
|
|
307
|
+
if not isinstance(block, dict):
|
|
308
|
+
return None
|
|
309
|
+
data = None
|
|
310
|
+
if block.get("type") in ("image", "document"):
|
|
311
|
+
source = block.get("source") or {}
|
|
312
|
+
if source.get("type") == "base64":
|
|
313
|
+
data = source.get("data")
|
|
314
|
+
else:
|
|
315
|
+
inline = block.get("inlineData") or block.get("inline_data")
|
|
316
|
+
if isinstance(inline, dict):
|
|
317
|
+
data = inline.get("data")
|
|
318
|
+
if not data:
|
|
319
|
+
return None
|
|
320
|
+
return hashlib.sha1(str(data).encode("ascii", errors="replace")).hexdigest()
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _has_function_response(content: Any) -> bool:
|
|
324
|
+
return isinstance(content, dict) and any(
|
|
325
|
+
isinstance(part, dict) and ("functionResponse" in part or "function_response" in part)
|
|
326
|
+
for part in content.get("parts") or [])
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def replaceable_tool_attachment(body: dict, path: list[str | int]) -> bool:
|
|
330
|
+
"""An attachment a tool returned, at a place where a text note is valid:
|
|
331
|
+
Anthropic `messages[i].content[j].content[k]` (inside a tool_result); Gemini
|
|
332
|
+
`contents[i].parts[j]` next to a functionResponse (Gemini CLI's sibling parts). OpenAI chat
|
|
333
|
+
tool messages carry text only, so tool images arrive as user messages: never changed."""
|
|
334
|
+
if len(path) == 6 and path[0] == "messages" and path[2] == "content" and path[4] == "content":
|
|
335
|
+
return True
|
|
336
|
+
if len(path) == 4 and path[0] == "contents" and path[2] == "parts":
|
|
337
|
+
return _has_function_response(body["contents"][path[1]])
|
|
338
|
+
return False
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def note_block(path: list[str | int], note: str) -> dict:
|
|
342
|
+
"""A text block in the provider's own format."""
|
|
343
|
+
return {"text": note} if path[0] == "contents" else {"type": "text", "text": note}
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def duplicate_note(number: int) -> str:
|
|
347
|
+
return (f"[TokenCut: identical to attachment #{number} earlier in this conversation "
|
|
348
|
+
"(same bytes), not sent again]")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
# ---------- step 5: shrink large images where the provider bills by size ----------
|
|
352
|
+
|
|
353
|
+
TARGET_LONG_EDGE = 1568 # what standard-tier Claude models always receive
|
|
354
|
+
MIN_SAVING = 0.10 # resize only if the provider's own formula saves at least this share
|
|
355
|
+
_RESIZE_FORMATS = {"PNG", "JPEG", "WEBP"}
|
|
356
|
+
_KEEP_DETAIL = re.compile(
|
|
357
|
+
r"\bzoom|\bpixel|\btiny\b|\bsmall (?:text|font|print)|\bfine print|\bread (?:the )?exact",
|
|
358
|
+
re.IGNORECASE)
|
|
359
|
+
_REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
|
|
360
|
+
_RESIZE_CACHE_MAX = 32
|
|
361
|
+
_resize_cache: dict[tuple, str | None] = {} # base64 of the cropped / resized image
|
|
362
|
+
_box_cache: dict[tuple[int, str, str], tuple[int, int, int, int] | None] = {}
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def tool_attachment(body: dict, path: list[str | int]) -> bool:
|
|
366
|
+
"""An attachment a tool returned (any provider format TokenCut can recognise)."""
|
|
367
|
+
if replaceable_tool_attachment(body, path):
|
|
368
|
+
return True
|
|
369
|
+
# Gemini CLI on multimodal models: contents[i].parts[j].functionResponse.parts[m]
|
|
370
|
+
return len(path) == 7 and path[0] == "contents" and path[5] == "parts"
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _tool_name(body: dict, path: list[str | int]) -> str:
|
|
374
|
+
if path[0] == "messages":
|
|
375
|
+
call_id = body["messages"][path[1]]["content"][path[3]].get("tool_use_id")
|
|
376
|
+
for message in body["messages"][:path[1]]:
|
|
377
|
+
for block in message.get("content") or [] if isinstance(message, dict) else []:
|
|
378
|
+
if isinstance(block, dict) and block.get("id") == call_id:
|
|
379
|
+
return str(block.get("name", ""))
|
|
380
|
+
return ""
|
|
381
|
+
for part in body["contents"][path[1]].get("parts") or []:
|
|
382
|
+
response = part.get("functionResponse") or part.get("function_response") if isinstance(
|
|
383
|
+
part, dict) else None
|
|
384
|
+
if isinstance(response, dict):
|
|
385
|
+
return str(response.get("name", ""))
|
|
386
|
+
return ""
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _computer_use(body: dict) -> bool:
|
|
390
|
+
return any(isinstance(t, dict) and (str(t.get("type", "")).startswith("computer")
|
|
391
|
+
or "computer" in str(t.get("name", "")).lower())
|
|
392
|
+
for t in body.get("tools") or [])
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def attachment_questions(segments: list[Segment]) -> dict[str, str]:
|
|
396
|
+
"""Per attachment segment: the user's last message before it plus the user text sent
|
|
397
|
+
with it (same message). Never a later message, so every decision based on it is the same
|
|
398
|
+
on every turn (prompt cache). Only role "user": agents such as Claude Code send their own
|
|
399
|
+
context (environment, skills list) as role "system" messages inside the conversation."""
|
|
400
|
+
with_message: dict[int, list[str]] = {}
|
|
401
|
+
for seg in segments:
|
|
402
|
+
if seg.kind == SegmentKind.USER_TEXT and seg.role == "user":
|
|
403
|
+
text = _REMINDER.sub("", seg.text).strip()
|
|
404
|
+
if text:
|
|
405
|
+
with_message.setdefault(seg.message_index, []).append(text)
|
|
406
|
+
questions, last = {}, ""
|
|
407
|
+
for seg in segments:
|
|
408
|
+
if seg.kind == SegmentKind.USER_TEXT and seg.role == "user":
|
|
409
|
+
text = _REMINDER.sub("", seg.text).strip()
|
|
410
|
+
if text:
|
|
411
|
+
last = text
|
|
412
|
+
elif seg.kind == SegmentKind.MEDIA:
|
|
413
|
+
questions[seg.id] = "\n".join([last, *with_message.get(seg.message_index, [])])
|
|
414
|
+
return questions
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _image_data(block: dict) -> tuple[str, str] | None:
|
|
418
|
+
"""(mime, base64 data) of an inline image block in any provider format."""
|
|
419
|
+
if block.get("type") == "image":
|
|
420
|
+
source = block.get("source") or {}
|
|
421
|
+
if source.get("type") == "base64":
|
|
422
|
+
return str(source.get("media_type", "")), str(source.get("data", ""))
|
|
423
|
+
return None
|
|
424
|
+
if block.get("type") == "image_url":
|
|
425
|
+
return _data_url(str((block.get("image_url") or {}).get("url", "")))
|
|
426
|
+
inline = block.get("inlineData") or block.get("inline_data")
|
|
427
|
+
if isinstance(inline, dict):
|
|
428
|
+
return str(inline.get("mimeType") or inline.get("mime_type") or ""), str(inline.get("data", ""))
|
|
429
|
+
return None
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _with_image_data(block: dict, data: str) -> dict:
|
|
433
|
+
new = copy.deepcopy(block)
|
|
434
|
+
if new.get("type") == "image":
|
|
435
|
+
new["source"]["data"] = data
|
|
436
|
+
elif new.get("type") == "image_url":
|
|
437
|
+
url = new["image_url"]["url"]
|
|
438
|
+
new["image_url"]["url"] = url[:url.index(",") + 1] + data
|
|
439
|
+
else:
|
|
440
|
+
key = "inlineData" if "inlineData" in new else "inline_data"
|
|
441
|
+
new[key]["data"] = data
|
|
442
|
+
return new
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _image_tokens(provider: str, model: str, w: int, h: int, block: dict, body: dict) -> int:
|
|
446
|
+
provider = pricing_provider(provider, model)
|
|
447
|
+
if provider == "anthropic":
|
|
448
|
+
return claude_image_tokens(w, h, model)
|
|
449
|
+
if provider == "openai":
|
|
450
|
+
return openai_image_tokens(w, h, model, str((block.get("image_url") or {}).get("detail", "auto")))
|
|
451
|
+
config = body.get("generationConfig") or body.get("generation_config") or {}
|
|
452
|
+
return gemini_image_tokens(w, h, model, str(config.get("mediaResolution")
|
|
453
|
+
or config.get("media_resolution") or ""))
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _adjusted_b64(data: str, box: tuple[int, int, int, int] | None,
|
|
457
|
+
size: tuple[int, int] | None) -> str | None:
|
|
458
|
+
"""Crop to `box`, then resize to `size`. Same input -> same output bytes (fixed filter and
|
|
459
|
+
settings), cached per image and operation."""
|
|
460
|
+
key = (len(data), data[:2048], data[-2048:], box, size)
|
|
461
|
+
if key not in _resize_cache:
|
|
462
|
+
if len(_resize_cache) >= _RESIZE_CACHE_MAX:
|
|
463
|
+
_resize_cache.pop(next(iter(_resize_cache)))
|
|
464
|
+
raw = _transform(_decode(data), box, size)
|
|
465
|
+
_resize_cache[key] = base64.b64encode(raw).decode() if raw is not None else None
|
|
466
|
+
return _resize_cache[key]
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _open_image(raw: bytes) -> Image.Image | None:
|
|
470
|
+
try:
|
|
471
|
+
img = Image.open(io.BytesIO(raw))
|
|
472
|
+
if img.format not in _RESIZE_FORMATS or getattr(img, "is_animated", False):
|
|
473
|
+
return None
|
|
474
|
+
img.load()
|
|
475
|
+
return img
|
|
476
|
+
except (OSError, ValueError, Image.DecompressionBombError):
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _transform(raw: bytes, box: tuple[int, int, int, int] | None,
|
|
481
|
+
size: tuple[int, int] | None) -> bytes | None:
|
|
482
|
+
img = _open_image(raw)
|
|
483
|
+
if img is None:
|
|
484
|
+
return None
|
|
485
|
+
fmt = img.format
|
|
486
|
+
out = img.crop(box) if box is not None else img
|
|
487
|
+
if size is not None and size != out.size:
|
|
488
|
+
out = out.resize(size, Image.Resampling.LANCZOS)
|
|
489
|
+
buf = io.BytesIO()
|
|
490
|
+
if fmt == "JPEG":
|
|
491
|
+
out.convert("RGB").save(buf, "JPEG", quality=90)
|
|
492
|
+
elif fmt == "WEBP":
|
|
493
|
+
out.save(buf, "WEBP", quality=90)
|
|
494
|
+
else:
|
|
495
|
+
out.save(buf, "PNG")
|
|
496
|
+
return buf.getvalue()
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
# ---------- step 6: plain outer borders are cut off ----------
|
|
500
|
+
|
|
501
|
+
BORDER_TOLERANCE = 16 # max per-channel difference still counted as background
|
|
502
|
+
BORDER_PAD = 8 # px of the border kept around the content
|
|
503
|
+
BORDER_MAX_REMOVED = 0.75 # cutting more than this share: a near-empty image, left alone
|
|
504
|
+
BORDER_MIN_CONTENT = 32 # px; smaller content boxes are left alone
|
|
505
|
+
_LAYOUT_QUESTION = re.compile(
|
|
506
|
+
r"\b(?:layout|align\w*|spacing|spaced|margins?|padding|cent(?:er|re)d?|position\w*|"
|
|
507
|
+
r"whitespace|white space|gaps?|offset|overflow\w*|cut off|borders?)\b", re.IGNORECASE)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def content_box(img: Image.Image) -> tuple[int, int, int, int] | None:
|
|
511
|
+
"""Box (left, top, right, bottom) around everything that differs from a plain background,
|
|
512
|
+
plus BORDER_PAD px; None if the corners do not share one colour or nothing differs."""
|
|
513
|
+
rgb = img.convert("RGB")
|
|
514
|
+
w, h = rgb.size
|
|
515
|
+
corners = [rgb.getpixel(p) for p in ((0, 0), (w - 1, 0), (0, h - 1), (w - 1, h - 1))]
|
|
516
|
+
background = corners[0]
|
|
517
|
+
if any(max(abs(a - b) for a, b in zip(c, background, strict=True)) > BORDER_TOLERANCE
|
|
518
|
+
for c in corners):
|
|
519
|
+
return None
|
|
520
|
+
red, green, blue = ImageChops.difference(rgb, Image.new("RGB", rgb.size, background)).split()
|
|
521
|
+
strongest = ImageChops.lighter(ImageChops.lighter(red, green), blue)
|
|
522
|
+
box = strongest.point(lambda v: 255 if v > BORDER_TOLERANCE else 0).getbbox()
|
|
523
|
+
if box is None:
|
|
524
|
+
return None
|
|
525
|
+
left, top, right, bottom = box
|
|
526
|
+
return (max(0, left - BORDER_PAD), max(0, top - BORDER_PAD),
|
|
527
|
+
min(w, right + BORDER_PAD), min(h, bottom + BORDER_PAD))
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _border_box(data: str, w: int, h: int) -> tuple[int, int, int, int] | None:
|
|
531
|
+
"""The content box worth cropping to, cached per image."""
|
|
532
|
+
key = (len(data), data[:2048], data[-2048:])
|
|
533
|
+
if key not in _box_cache:
|
|
534
|
+
if len(_box_cache) >= _RESIZE_CACHE_MAX:
|
|
535
|
+
_box_cache.pop(next(iter(_box_cache)))
|
|
536
|
+
img = _open_image(_decode(data))
|
|
537
|
+
box = content_box(img) if img is not None else None
|
|
538
|
+
if box is not None:
|
|
539
|
+
bw, bh = box[2] - box[0], box[3] - box[1]
|
|
540
|
+
if (bw, bh) == (w, h) or min(bw, bh) < BORDER_MIN_CONTENT \
|
|
541
|
+
or 1 - (bw * bh) / (w * h) > BORDER_MAX_REMOVED:
|
|
542
|
+
box = None
|
|
543
|
+
_box_cache[key] = box
|
|
544
|
+
return _box_cache[key]
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _image_note(w: int, h: int, box: tuple[int, int, int, int], size: tuple[int, int]) -> str:
|
|
548
|
+
bw, bh = box[2] - box[0], box[3] - box[1]
|
|
549
|
+
scaled = f", then scaled to {size[0]}x{size[1]}" if size != (bw, bh) else ""
|
|
550
|
+
return (f"[TokenCut: plain border cut from this image: original {w}x{h}, kept the "
|
|
551
|
+
f"{bw}x{bh} area at offset ({box[0]}, {box[1]}){scaled}; positions refer to the "
|
|
552
|
+
"image as sent]")
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _scaled_note(w: int, h: int, size: tuple[int, int]) -> str:
|
|
556
|
+
return (f"[TokenCut: image scaled from {w}x{h} to {size[0]}x{size[1]} to save tokens; "
|
|
557
|
+
"positions refer to the image as sent]")
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def adjust_images(provider: str, model: str, body: dict, segments: list[Segment],
|
|
561
|
+
include_user: bool = False, resize: bool = True, trim: bool = True
|
|
562
|
+
) -> tuple[list[tuple[list[str | int], list[dict]]], list[dict]]:
|
|
563
|
+
"""(path -> replacement blocks, report entries): plain borders cut (step 6) and large images
|
|
564
|
+
shrunk (step 5), only where the provider's own formula saves >= MIN_SAVING. The body is not
|
|
565
|
+
changed; a cropped image is followed by a note, so insert in descending path order."""
|
|
566
|
+
if _computer_use(body) or not (resize or trim):
|
|
567
|
+
return [], []
|
|
568
|
+
questions = attachment_questions(segments)
|
|
569
|
+
changes, entries = [], []
|
|
570
|
+
for seg in segments:
|
|
571
|
+
if seg.kind != SegmentKind.MEDIA:
|
|
572
|
+
continue
|
|
573
|
+
block = lookup(body, seg.path)
|
|
574
|
+
image = _image_data(block) if isinstance(block, dict) else None
|
|
575
|
+
if image is None or not image[0].startswith("image/"):
|
|
576
|
+
continue
|
|
577
|
+
is_tool = tool_attachment(body, seg.path)
|
|
578
|
+
if not (is_tool or include_user):
|
|
579
|
+
continue
|
|
580
|
+
if is_tool and "computer" in _tool_name(body, seg.path).lower():
|
|
581
|
+
continue
|
|
582
|
+
size = image_size(_decode(image[1], HEAD_B64_CHARS))
|
|
583
|
+
if size is None:
|
|
584
|
+
continue
|
|
585
|
+
w, h = size
|
|
586
|
+
# a note can follow the image except inside Gemini's functionResponse.parts
|
|
587
|
+
can_note = not (len(seg.path) == 7 and seg.path[0] == "contents")
|
|
588
|
+
question = questions.get(seg.id, "")
|
|
589
|
+
trim_here = trim and can_note and not _LAYOUT_QUESTION.search(question)
|
|
590
|
+
resize_here = resize and not _KEEP_DETAIL.search(question)
|
|
591
|
+
box = _border_box(image[1], w, h) if trim_here else None
|
|
592
|
+
cw, ch = (box[2] - box[0], box[3] - box[1]) if box is not None else (w, h)
|
|
593
|
+
target = (cw, ch)
|
|
594
|
+
if resize_here and max(cw, ch) > TARGET_LONG_EDGE:
|
|
595
|
+
scale = TARGET_LONG_EDGE / max(cw, ch)
|
|
596
|
+
target = (max(1, round(cw * scale)), max(1, round(ch * scale)))
|
|
597
|
+
if box is None and target == (w, h):
|
|
598
|
+
continue
|
|
599
|
+
before = _image_tokens(provider, model, w, h, block, body)
|
|
600
|
+
after = _image_tokens(provider, model, *target, block, body)
|
|
601
|
+
if before - after < MIN_SAVING * before:
|
|
602
|
+
continue
|
|
603
|
+
encoded = _adjusted_b64(image[1], box, target)
|
|
604
|
+
if encoded is None:
|
|
605
|
+
continue
|
|
606
|
+
blocks = [_with_image_data(block, encoded)]
|
|
607
|
+
if box is not None:
|
|
608
|
+
blocks.append(note_block(seg.path, _image_note(w, h, box, target)))
|
|
609
|
+
elif can_note:
|
|
610
|
+
blocks.append(note_block(seg.path, _scaled_note(w, h, target)))
|
|
611
|
+
how = ("border_trimmed_resized" if target != (cw, ch) else "border_trimmed") \
|
|
612
|
+
if box is not None else "resized"
|
|
613
|
+
changes.append((seg.path, blocks))
|
|
614
|
+
entries.append({"segment": seg.id, "how": how, "tool": "", "kind": "image",
|
|
615
|
+
"from_size": [w, h], "to_size": list(target),
|
|
616
|
+
"crop_box": list(box) if box is not None else None,
|
|
617
|
+
"before": before,
|
|
618
|
+
"after": after})
|
|
619
|
+
return changes, entries
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
# ---------- step 7: PDF pages without images or drawings are sent as text ----------
|
|
623
|
+
|
|
624
|
+
PDF_MIN_TEXT_CHARS = 200 # less text than this: probably a scan or a figure page
|
|
625
|
+
PDF_MAX_DRAW_OPS = 20 # more line / box / curve operators: probably a table or a chart
|
|
626
|
+
PDF_TABLE_ROWS = 3 # this many number-table rows: a table (columns are lost as plain text)
|
|
627
|
+
PDF_MAX_PAGES = 50 # longer PDFs are left as they are (~115 ms per page the first time)
|
|
628
|
+
_DRAW_OP = re.compile(rb"(?<![A-Za-z])(?:re|l|c|v|y)(?=\s)")
|
|
629
|
+
_NUMBER_CELL = re.compile(r"[-+]?\d+(?:[.,]\d+)*%?")
|
|
630
|
+
# Intent safety: a question about how pages look keeps the whole PDF (EcoDoc, ACL 2025, also
|
|
631
|
+
# chooses text or image per page from page characteristics and query intent).
|
|
632
|
+
_VISUAL_QUESTION = re.compile(
|
|
633
|
+
r"\b(?:charts?|graphs?|plots?|figures?|fig\.|diagrams?|images?|pictures?|photos?|"
|
|
634
|
+
r"logos?|signatures?|signed|stamps?|seals?|handwrit\w*|colou?rs?|layout|formatting|"
|
|
635
|
+
r"(?<!answer )formats?(?!:|\s+(?:your|the|this)\s+(?:final\s+)?(?:answer|reply|response))|"
|
|
636
|
+
r"fonts?|scan(?:ned|s)?|tables?|maps?|drawings?|screenshots?|visual\w*|"
|
|
637
|
+
r"looks? like|appearance|highlight\w*)", re.IGNORECASE)
|
|
638
|
+
_PDF_ERRORS = (PyPdfError, ValueError, KeyError, TypeError, AttributeError, OSError)
|
|
639
|
+
_SPLIT_CACHE_MAX = 32
|
|
640
|
+
_split_cache: dict[tuple[int, str, str], tuple[str, str | None, list[int], int] | None] = {}
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _draw_ops(page: Any) -> int:
|
|
644
|
+
"""Line / box / curve operators on the page and in its form XObjects (vector charts)."""
|
|
645
|
+
contents = page.get_contents()
|
|
646
|
+
total = len(_DRAW_OP.findall(contents.get_data())) if contents is not None else 0
|
|
647
|
+
resources = page.get("/Resources")
|
|
648
|
+
xobjects = resources.get_object().get("/XObject") if resources is not None else None
|
|
649
|
+
for ref in (xobjects.get_object().values() if xobjects is not None else []):
|
|
650
|
+
obj = ref.get_object()
|
|
651
|
+
if obj.get("/Subtype") == "/Form":
|
|
652
|
+
total += len(_DRAW_OP.findall(obj.get_data()))
|
|
653
|
+
return total
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _number_table_rows(page: Any) -> int:
|
|
657
|
+
"""Rows with >= 4 columns of which >= 3 are numbers, in layout-preserving text."""
|
|
658
|
+
rows = 0
|
|
659
|
+
for line in (page.extract_text(extraction_mode="layout") or "").splitlines():
|
|
660
|
+
cells = [c for c in re.split(r" {2,}", line.strip()) if c]
|
|
661
|
+
if len(cells) >= 4 and sum(1 for c in cells if _NUMBER_CELL.fullmatch(c)) >= 3:
|
|
662
|
+
rows += 1
|
|
663
|
+
return rows
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def pdf_page_kind(page: Any) -> tuple[str, str]:
|
|
667
|
+
"""("text" / "visual", page text). A page is "text" only with enough text, no raster
|
|
668
|
+
images, almost no drawings and no number-table rows; scans, figures, charts and tables
|
|
669
|
+
stay "visual". Raises one of _PDF_ERRORS on an unreadable page."""
|
|
670
|
+
text = (page.extract_text() or "").strip()
|
|
671
|
+
visual = (len(page.images) > 0 or len(text) < PDF_MIN_TEXT_CHARS
|
|
672
|
+
or _draw_ops(page) > PDF_MAX_DRAW_OPS
|
|
673
|
+
or _number_table_rows(page) >= PDF_TABLE_ROWS)
|
|
674
|
+
return ("visual" if visual else "text"), text
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _analyse_pdf(raw: bytes) -> tuple[Any, list[str], list[str]] | None:
|
|
678
|
+
"""(reader, per-page "text" / "visual", per-page text), or None if unreadable / too long."""
|
|
679
|
+
try:
|
|
680
|
+
reader = PdfReader(io.BytesIO(raw))
|
|
681
|
+
if reader.is_encrypted or len(reader.pages) > PDF_MAX_PAGES:
|
|
682
|
+
return None
|
|
683
|
+
kinds, texts = [], []
|
|
684
|
+
for page in reader.pages:
|
|
685
|
+
kind, text = pdf_page_kind(page)
|
|
686
|
+
kinds.append(kind)
|
|
687
|
+
texts.append(text)
|
|
688
|
+
return reader, kinds, texts
|
|
689
|
+
except _PDF_ERRORS:
|
|
690
|
+
return None
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def pdf_page_kinds(raw: bytes) -> list[str] | None:
|
|
694
|
+
analysed = _analyse_pdf(raw)
|
|
695
|
+
return analysed[1] if analysed else None
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _split_pdf(data: str) -> tuple[str, str | None, list[int], int] | None:
|
|
699
|
+
"""(text of the text pages, base64 PDF of the other pages or None, text page numbers,
|
|
700
|
+
page count), or None if there is nothing to send as text. Same bytes every time."""
|
|
701
|
+
key = (len(data), data[:2048], data[-2048:])
|
|
702
|
+
if key in _split_cache:
|
|
703
|
+
return _split_cache[key]
|
|
704
|
+
result = None
|
|
705
|
+
analysed = _analyse_pdf(_decode(data))
|
|
706
|
+
if analysed and "text" in analysed[1]:
|
|
707
|
+
reader, kinds, texts = analysed
|
|
708
|
+
try:
|
|
709
|
+
text_pages = [n for n, kind in enumerate(kinds) if kind == "text"]
|
|
710
|
+
text = "\n\n".join(f"--- page {n + 1} ---\n{texts[n]}" for n in text_pages)
|
|
711
|
+
visual = [n for n, kind in enumerate(kinds) if kind == "visual"]
|
|
712
|
+
subset = None
|
|
713
|
+
if visual:
|
|
714
|
+
writer = PdfWriter()
|
|
715
|
+
for n in visual:
|
|
716
|
+
writer.add_page(reader.pages[n])
|
|
717
|
+
buf = io.BytesIO()
|
|
718
|
+
writer.write(buf)
|
|
719
|
+
subset = base64.b64encode(buf.getvalue()).decode()
|
|
720
|
+
result = (text, subset, [n + 1 for n in text_pages], len(kinds))
|
|
721
|
+
except _PDF_ERRORS:
|
|
722
|
+
result = None
|
|
723
|
+
if len(_split_cache) >= _SPLIT_CACHE_MAX:
|
|
724
|
+
_split_cache.pop(next(iter(_split_cache)))
|
|
725
|
+
_split_cache[key] = result
|
|
726
|
+
return result
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _read_pdf_tool(body: dict) -> str:
|
|
730
|
+
"""Name of the agent's read_pdf tool (tokencut/pdftool), or "". The tool list starts the
|
|
731
|
+
cached prefix, so a note based on it never changes an earlier turn on its own."""
|
|
732
|
+
for tool in body.get("tools") or []:
|
|
733
|
+
if isinstance(tool, dict):
|
|
734
|
+
name = str(tool.get("name") or (tool.get("function") or {}).get("name") or "")
|
|
735
|
+
if name.endswith("read_pdf"):
|
|
736
|
+
return name
|
|
737
|
+
return ""
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def _pdf_note(name: str, text_pages: list[int], total: int, visual_left: bool,
|
|
741
|
+
reader: str = "") -> str:
|
|
742
|
+
pages = ", ".join(str(n) for n in text_pages)
|
|
743
|
+
rest = "the other pages follow as PDF" if visual_left else "no page images sent"
|
|
744
|
+
label = f" '{name}'" if name else ""
|
|
745
|
+
ask = (f"; to see one of these pages as a picture, call {reader} with as_images=true"
|
|
746
|
+
if reader else "")
|
|
747
|
+
return (f"[TokenCut: PDF{label}, pages {pages} of {total} sent as text (no images or "
|
|
748
|
+
f"drawings found on them); {rest}{ask}]")
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def split_pdfs(provider: str, body: dict, segments: list[Segment], include_user: bool = False
|
|
752
|
+
) -> tuple[list[tuple[list[str | int], list[dict]]], list[dict]]:
|
|
753
|
+
"""(path -> replacement blocks, report entries) for PDFs with text pages (Claude, OpenAI).
|
|
754
|
+
Insert the replacements in descending path order: one block may become two."""
|
|
755
|
+
if provider not in ("anthropic", "openai"):
|
|
756
|
+
return [], [] # Gemini: its docs conflict on whether PDF text is billed
|
|
757
|
+
changes, entries = [], []
|
|
758
|
+
questions = attachment_questions(segments)
|
|
759
|
+
reader = _read_pdf_tool(body)
|
|
760
|
+
for seg in segments:
|
|
761
|
+
if seg.kind != SegmentKind.MEDIA:
|
|
762
|
+
continue
|
|
763
|
+
block = lookup(body, seg.path)
|
|
764
|
+
if not isinstance(block, dict):
|
|
765
|
+
continue
|
|
766
|
+
if not (tool_attachment(body, seg.path) or include_user):
|
|
767
|
+
continue
|
|
768
|
+
if _VISUAL_QUESTION.search(questions.get(seg.id, "")):
|
|
769
|
+
continue # the question is about how pages look: keep every page picture
|
|
770
|
+
if provider == "anthropic" and block.get("type") == "document":
|
|
771
|
+
source = block.get("source") or {}
|
|
772
|
+
if source.get("type") != "base64" or source.get("media_type") != "application/pdf":
|
|
773
|
+
continue
|
|
774
|
+
data, name = str(source.get("data", "")), str(block.get("title", ""))
|
|
775
|
+
elif provider == "openai" and block.get("type") == "file":
|
|
776
|
+
parsed = _data_url(str((block.get("file") or {}).get("file_data", "")))
|
|
777
|
+
if parsed is None or parsed[0] != "application/pdf":
|
|
778
|
+
continue
|
|
779
|
+
data, name = parsed[1], str((block.get("file") or {}).get("filename", ""))
|
|
780
|
+
else:
|
|
781
|
+
continue
|
|
782
|
+
split = _split_pdf(data)
|
|
783
|
+
if split is None:
|
|
784
|
+
continue
|
|
785
|
+
text, subset, text_pages, total = split
|
|
786
|
+
note = _pdf_note(name, text_pages, total, subset is not None, reader)
|
|
787
|
+
blocks = [{"type": "text", "text": f"{note}\n\n{text}"}]
|
|
788
|
+
if subset is not None:
|
|
789
|
+
rest = copy.deepcopy(block)
|
|
790
|
+
if provider == "anthropic":
|
|
791
|
+
rest["source"]["data"] = subset
|
|
792
|
+
else:
|
|
793
|
+
rest["file"]["file_data"] = "data:application/pdf;base64," + subset
|
|
794
|
+
blocks.append(rest)
|
|
795
|
+
elif "cache_control" in block:
|
|
796
|
+
blocks[0]["cache_control"] = block["cache_control"]
|
|
797
|
+
changes.append((seg.path, blocks))
|
|
798
|
+
entries.append({"segment": seg.id, "how": "pdf_text_pages", "tool": "",
|
|
799
|
+
"before": seg.token_count, "after": seg.token_count, "pages": total,
|
|
800
|
+
"page_images_removed": len(text_pages), "kind": "pdf",
|
|
801
|
+
"text_pages": text_pages,
|
|
802
|
+
"visual_pages": [n for n in range(1, total + 1) if n not in text_pages]})
|
|
803
|
+
return changes, entries
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def with_media_tokens(provider: str, model: str, body: dict,
|
|
807
|
+
segments: list[Segment]) -> tuple[list[Segment], dict]:
|
|
808
|
+
"""Segments with estimated attachment tokens, and counts for the analysis."""
|
|
809
|
+
stats = {"images": 0, "documents": 0, "unknown": 0, "tokens": 0}
|
|
810
|
+
out = []
|
|
811
|
+
for seg in segments:
|
|
812
|
+
found = None
|
|
813
|
+
if seg.kind == SegmentKind.MEDIA:
|
|
814
|
+
found = _estimate(provider, model, lookup(body, seg.path), body)
|
|
815
|
+
if found is None:
|
|
816
|
+
out.append(seg)
|
|
817
|
+
continue
|
|
818
|
+
kind, tokens = found
|
|
819
|
+
stats["images" if kind == "image" else "documents"] += 1
|
|
820
|
+
if tokens is None:
|
|
821
|
+
stats["unknown"] += 1
|
|
822
|
+
tokens = 0
|
|
823
|
+
stats["tokens"] += tokens
|
|
824
|
+
out.append(seg.model_copy(update={"token_count": tokens}))
|
|
825
|
+
return out, stats
|