tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Cache-safe trimming of a provider request (Iteration 2, phase 3).
|
|
2
|
+
|
|
3
|
+
Only large tool results are trimmed: file contents, command output, search results.
|
|
4
|
+
System prompt, tool definitions, user and assistant text, thinking and tool calls are
|
|
5
|
+
never touched, and every tool call keeps its result (only the result's text gets shorter).
|
|
6
|
+
|
|
7
|
+
Cache safety: providers bill a repeated prompt prefix at ~10% of the price, and changing
|
|
8
|
+
anything inside that prefix forces a full-price rewrite. So the trim of a tool result
|
|
9
|
+
depends ONLY on that result and on what came before it in the conversation (the user's
|
|
10
|
+
request, the tool call that produced it) — never on later messages or on the current
|
|
11
|
+
turn. A result is therefore trimmed identically on every later turn and the cached
|
|
12
|
+
prefix stays byte-for-byte the same. (tests/test_gateway_trimming.py checks this.)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import copy
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from pathlib import PurePosixPath
|
|
20
|
+
|
|
21
|
+
from tokencut.contracts import TIER_BUDGET_FRACTION, Segment, SegmentKind, Tier, ToolResultKind
|
|
22
|
+
from tokencut.gateway.adapters.common import lookup, set_text
|
|
23
|
+
from tokencut.gateway.ccr import with_ref
|
|
24
|
+
from tokencut.gateway.codeview import trim_file_listing
|
|
25
|
+
from tokencut.gateway.media import (
|
|
26
|
+
attachment_key,
|
|
27
|
+
duplicate_note,
|
|
28
|
+
note_block,
|
|
29
|
+
replaceable_tool_attachment,
|
|
30
|
+
)
|
|
31
|
+
from tokencut.gateway.textcut import looks_like_table, trim_output
|
|
32
|
+
from tokencut.tokens import count_tokens
|
|
33
|
+
|
|
34
|
+
_GATEWAY_TOOL_PREFIX = "tokencut_" # tools the gateway answers itself (virtual.py)
|
|
35
|
+
_REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
|
|
36
|
+
_WORD = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|[0-9]+")
|
|
37
|
+
_EDIT_WORDS = frozenset({"edit", "write", "replace", "create", "patch", "insert"})
|
|
38
|
+
_PATH_KEYS = ("file_path", "path", "absolute_path", "filename", "target_file", "file")
|
|
39
|
+
# `cat foo.py`, `head -n 50 foo.py`, `sed -n '1,80p' foo.py`, `nl foo.py` — one file only
|
|
40
|
+
_SHOW_FILE = re.compile(
|
|
41
|
+
r"^\s*(?:cat|head|tail|nl|bat|sed\s+-n\s+\S+)\b(?:\s+-\S+)*(?:\s+\d+)?\s+"
|
|
42
|
+
r"([^\s|;&<>]+\.py)\s*$"
|
|
43
|
+
)
|
|
44
|
+
# R8: a diff is code, not a log (every line matters), so it is never trimmed on arrival.
|
|
45
|
+
_DIFF = re.compile(r"^(?:diff --git |@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@)", re.MULTILINE)
|
|
46
|
+
_TABLE_FILE = re.compile(r"\.(?:csv|tsv)\b", re.IGNORECASE)
|
|
47
|
+
_PDF_FILE = re.compile(r"\.pdf\b", re.IGNORECASE)
|
|
48
|
+
# R8: "show me the full diff / file / output ..." -> the user wants that output whole.
|
|
49
|
+
# "the whole test suite" does not match: the word must refer to an output-like noun, in the
|
|
50
|
+
# same phrase (only spaces between): "wait for it to complete, see its output" is no request.
|
|
51
|
+
_FULL_REQUEST = re.compile(
|
|
52
|
+
r"\b(?:full|entire|whole|complete)\b(?:[ \t]+[\w'-]+){0,2}?[ \t]+"
|
|
53
|
+
r"(?:diffs?|files?|output|contents?|code|logs?|listing)\b",
|
|
54
|
+
re.IGNORECASE,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class TrimConfig:
|
|
60
|
+
tier: Tier = Tier.MODERATE
|
|
61
|
+
min_tokens: int = 2_000 # tool results smaller than this are never trimmed
|
|
62
|
+
min_keep_tokens: int = 500 # never cut a result below this
|
|
63
|
+
ccr: bool = False # add a reference to every shortened output (it can be fetched back)
|
|
64
|
+
resize_images: bool = True # attachments step 5: shrink large images where it saves tokens
|
|
65
|
+
trim_borders: bool = True # attachments step 6: cut plain outer borders off images
|
|
66
|
+
media_user: bool = False # also change the user's own attachments (team decision; off)
|
|
67
|
+
pdf_text_pages: bool = True # attachments step 7: PDF pages without images / drawings as text
|
|
68
|
+
system_note: bool = False # add the fixed note on TokenCut's markers to the system prompt
|
|
69
|
+
prompt_brief: bool = False # add a task brief to typed user messages (brief.py, opt-in)
|
|
70
|
+
brief_threshold: float = 0.87 # router confidence needed for a brief (chosen on dev)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class TrimReport:
|
|
75
|
+
tokens_before: int = 0
|
|
76
|
+
tokens_after: int = 0
|
|
77
|
+
trimmed: list[dict] = field(default_factory=list) # one entry per trimmed segment
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def saved(self) -> int:
|
|
81
|
+
return self.tokens_before - self.tokens_after
|
|
82
|
+
|
|
83
|
+
def as_dict(self) -> dict:
|
|
84
|
+
return {
|
|
85
|
+
"segments_trimmed": sum(1 for t in self.trimmed
|
|
86
|
+
if t["how"] not in ("ccr_tool", "system_note", "task_brief")),
|
|
87
|
+
"tokens_before": self.tokens_before,
|
|
88
|
+
"tokens_after": self.tokens_after,
|
|
89
|
+
"tokens_saved": self.saved,
|
|
90
|
+
"details": self.trimmed,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass
|
|
95
|
+
class _Context:
|
|
96
|
+
"""What is known at a given point of the conversation (only from earlier segments)."""
|
|
97
|
+
|
|
98
|
+
last_user_text: str = ""
|
|
99
|
+
user_texts: list[str] = field(default_factory=list)
|
|
100
|
+
calls: dict[str, tuple[str, str]] = field(default_factory=dict) # call id -> (name, input)
|
|
101
|
+
last_call_by_name: dict[str, str] = field(default_factory=dict) # for Gemini (no ids)
|
|
102
|
+
edited: set[str] = field(default_factory=set) # paths an edit tool has touched
|
|
103
|
+
attachments: dict[str, int] = field(default_factory=dict) # content hash -> attachment number
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _words(name: str) -> set[str]:
|
|
107
|
+
return {w.lower() for w in _WORD.findall(name)}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _call_path(call_input: str) -> str:
|
|
111
|
+
"""The file a tool call is about, if any (from its JSON input)."""
|
|
112
|
+
try:
|
|
113
|
+
data = json.loads(call_input)
|
|
114
|
+
except ValueError:
|
|
115
|
+
return ""
|
|
116
|
+
if not isinstance(data, dict):
|
|
117
|
+
return ""
|
|
118
|
+
for key in _PATH_KEYS:
|
|
119
|
+
if isinstance(data.get(key), str):
|
|
120
|
+
return data[key]
|
|
121
|
+
command = data.get("command") or data.get("cmd")
|
|
122
|
+
if isinstance(command, list):
|
|
123
|
+
command = " ".join(str(part) for part in command)
|
|
124
|
+
if isinstance(command, str):
|
|
125
|
+
match = _SHOW_FILE.match(command)
|
|
126
|
+
if match:
|
|
127
|
+
return match.group(1)
|
|
128
|
+
return ""
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _is_edit_call(name: str, call_input: str) -> bool:
|
|
132
|
+
if '"command": "view"' in call_input: # editor tools that also have a read-only command
|
|
133
|
+
return False
|
|
134
|
+
return bool(_words(name) & _EDIT_WORDS)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _mentioned(path: str, texts: list[str]) -> bool:
|
|
138
|
+
"""Did the user name this file? (then it is probably about to be edited: keep it whole)"""
|
|
139
|
+
p = PurePosixPath(path.replace("\\", "/"))
|
|
140
|
+
tail = "/".join(p.parts[-2:]) if len(p.parts) >= 2 else p.name
|
|
141
|
+
pattern = re.compile(rf"(?<![\w.]){re.escape(p.name)}(?![\w])")
|
|
142
|
+
return any(tail in text or pattern.search(text) for text in texts)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _same_file(a: str, b: str) -> bool:
|
|
146
|
+
a, b = a.replace("\\", "/"), b.replace("\\", "/")
|
|
147
|
+
return a == b or a.endswith("/" + b) or b.endswith("/" + a)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def trim_request(
|
|
151
|
+
body: dict, segments: list[Segment], config: TrimConfig
|
|
152
|
+
) -> tuple[dict | None, TrimReport]:
|
|
153
|
+
"""Return (new body or None if nothing changed, report)."""
|
|
154
|
+
report = TrimReport()
|
|
155
|
+
ctx = _Context()
|
|
156
|
+
new_body: dict | None = None
|
|
157
|
+
|
|
158
|
+
for seg in segments: # adapters emit segments in request order
|
|
159
|
+
if seg.kind == SegmentKind.USER_TEXT:
|
|
160
|
+
text = _REMINDER.sub("", seg.text).strip()
|
|
161
|
+
if text:
|
|
162
|
+
ctx.last_user_text = text
|
|
163
|
+
ctx.user_texts.append(text)
|
|
164
|
+
elif seg.kind == SegmentKind.TOOL_CALL:
|
|
165
|
+
ctx.calls[seg.tool_call_id] = (seg.tool_name, seg.text)
|
|
166
|
+
ctx.last_call_by_name[seg.tool_name] = seg.tool_call_id
|
|
167
|
+
if _is_edit_call(seg.tool_name, seg.text):
|
|
168
|
+
path = _call_path(seg.text)
|
|
169
|
+
if path:
|
|
170
|
+
ctx.edited.add(path)
|
|
171
|
+
elif seg.kind == SegmentKind.MEDIA:
|
|
172
|
+
# Attachments plan step 4: a tool returning bytes already in the conversation (an
|
|
173
|
+
# unchanged screenshot) gets a note instead. Decided from earlier content only, so it
|
|
174
|
+
# is identical every turn; the user's own attachments are never changed.
|
|
175
|
+
key = attachment_key(lookup(body, seg.path))
|
|
176
|
+
if key is None:
|
|
177
|
+
continue
|
|
178
|
+
if key not in ctx.attachments:
|
|
179
|
+
ctx.attachments[key] = len(ctx.attachments) + 1
|
|
180
|
+
continue
|
|
181
|
+
if not replaceable_tool_attachment(body, seg.path):
|
|
182
|
+
continue
|
|
183
|
+
note = duplicate_note(ctx.attachments[key])
|
|
184
|
+
if new_body is None:
|
|
185
|
+
new_body = copy.deepcopy(body)
|
|
186
|
+
lookup(new_body, seg.path[:-1])[seg.path[-1]] = note_block(seg.path, note)
|
|
187
|
+
after = count_tokens(note)
|
|
188
|
+
report.tokens_before += seg.token_count
|
|
189
|
+
report.tokens_after += after
|
|
190
|
+
report.trimmed.append({"segment": seg.id, "how": "duplicate_attachment",
|
|
191
|
+
"tool": seg.tool_name, "before": seg.token_count,
|
|
192
|
+
"after": after})
|
|
193
|
+
elif seg.kind == SegmentKind.TOOL_RESULT and seg.token_count >= config.min_tokens:
|
|
194
|
+
if seg.tool_name.startswith(_GATEWAY_TOOL_PREFIX):
|
|
195
|
+
continue # the gateway's own answers (a retrieved original) stay whole
|
|
196
|
+
new_text, how = _trim_result(seg, ctx, config)
|
|
197
|
+
if new_text is None or not isinstance(lookup(body, seg.path), str):
|
|
198
|
+
continue
|
|
199
|
+
if config.ccr:
|
|
200
|
+
new_text = with_ref(new_text, seg.text)
|
|
201
|
+
after = count_tokens(new_text)
|
|
202
|
+
if after >= seg.token_count:
|
|
203
|
+
continue
|
|
204
|
+
if new_body is None:
|
|
205
|
+
new_body = copy.deepcopy(body)
|
|
206
|
+
set_text(new_body, seg.path, new_text)
|
|
207
|
+
report.tokens_before += seg.token_count
|
|
208
|
+
report.tokens_after += after
|
|
209
|
+
report.trimmed.append(
|
|
210
|
+
{
|
|
211
|
+
"segment": seg.id,
|
|
212
|
+
"how": how,
|
|
213
|
+
"tool": seg.tool_name,
|
|
214
|
+
"before": seg.token_count,
|
|
215
|
+
"after": after,
|
|
216
|
+
}
|
|
217
|
+
)
|
|
218
|
+
return new_body, report
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _trim_result(seg: Segment, ctx: _Context, config: TrimConfig) -> tuple[str | None, str]:
|
|
222
|
+
call_id = seg.tool_call_id or ctx.last_call_by_name.get(seg.tool_name, "")
|
|
223
|
+
_, call_input = ctx.calls.get(call_id, (seg.tool_name, ""))
|
|
224
|
+
path = _call_path(call_input)
|
|
225
|
+
budget = max(config.min_keep_tokens, round(TIER_BUDGET_FRACTION[config.tier] * seg.token_count))
|
|
226
|
+
query = f"{ctx.last_user_text}\n{call_input}"
|
|
227
|
+
|
|
228
|
+
# R8: trimming a diff, or output the user asked for in full, leaves fragments the agent
|
|
229
|
+
# fetches again (live run L1: 3 extra requests, a net loss). Leave them whole.
|
|
230
|
+
if _DIFF.search(seg.text) or _FULL_REQUEST.search(ctx.last_user_text):
|
|
231
|
+
return None, "protected"
|
|
232
|
+
# Tables: head / tail would drop the middle rows, where the answer may be.
|
|
233
|
+
if seg.result_kind in (ToolResultKind.FILE_READ, ToolResultKind.COMMAND_OUTPUT) and (
|
|
234
|
+
_TABLE_FILE.search(path or call_input) or looks_like_table(seg.text)
|
|
235
|
+
):
|
|
236
|
+
return None, "protected"
|
|
237
|
+
# PDF text (read_pdf, tokencut/pdftool): only the text pages, already chosen; head / tail
|
|
238
|
+
# would drop middle pages.
|
|
239
|
+
if seg.result_kind == ToolResultKind.FILE_READ and _PDF_FILE.search(path or call_input):
|
|
240
|
+
return None, "protected"
|
|
241
|
+
|
|
242
|
+
if path.endswith(".py") and seg.result_kind in (
|
|
243
|
+
ToolResultKind.FILE_READ,
|
|
244
|
+
ToolResultKind.COMMAND_OUTPUT,
|
|
245
|
+
):
|
|
246
|
+
# Edit safety: files the user named or the agent already edited stay whole.
|
|
247
|
+
if any(_same_file(path, e) for e in ctx.edited) or _mentioned(path, ctx.user_texts):
|
|
248
|
+
return None, "protected"
|
|
249
|
+
return trim_file_listing(seg.text, path, query, config.tier), "code"
|
|
250
|
+
if seg.result_kind == ToolResultKind.FILE_READ:
|
|
251
|
+
return trim_output(seg.text, budget, "file content"), "file_head_tail"
|
|
252
|
+
if seg.result_kind == ToolResultKind.COMMAND_OUTPUT:
|
|
253
|
+
return trim_output(seg.text, budget, "command output"), "command_output"
|
|
254
|
+
if seg.result_kind == ToolResultKind.SEARCH:
|
|
255
|
+
return trim_output(seg.text, budget, "search results"), "search"
|
|
256
|
+
return None, "unknown_tool"
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Read the provider's own token usage from a response while it streams past.
|
|
2
|
+
|
|
3
|
+
Works on raw bytes as they are forwarded; never changes them. Streaming responses
|
|
4
|
+
(text/event-stream) are scanned line by line; plain JSON bodies are parsed at the end.
|
|
5
|
+
Gives real billed numbers, including cached input, which the router's cost model needs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import asdict, dataclass
|
|
10
|
+
|
|
11
|
+
MAX_BUFFER_BYTES = 10_000_000 # plain JSON bodies larger than this are not parsed
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Usage:
|
|
16
|
+
input_tokens: int = 0 # uncached input
|
|
17
|
+
cache_read_tokens: int = 0
|
|
18
|
+
cache_write_tokens: int = 0
|
|
19
|
+
output_tokens: int = 0
|
|
20
|
+
|
|
21
|
+
def as_dict(self) -> dict[str, int]:
|
|
22
|
+
return asdict(self)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _int(value: object) -> int:
|
|
26
|
+
return value if isinstance(value, int) else 0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def apply_usage(provider: str, event: object, usage: Usage) -> None:
|
|
30
|
+
"""Update `usage` from one JSON object (an SSE event or a whole response body)."""
|
|
31
|
+
if isinstance(event, list): # Gemini non-SSE streams are a JSON array of chunks
|
|
32
|
+
for item in event:
|
|
33
|
+
apply_usage(provider, item, usage)
|
|
34
|
+
return
|
|
35
|
+
if not isinstance(event, dict):
|
|
36
|
+
return
|
|
37
|
+
if provider == "anthropic":
|
|
38
|
+
# message_start carries input + cache numbers; message_delta the final output count
|
|
39
|
+
data = (
|
|
40
|
+
event.get("message", {}).get("usage") if event.get("type") == "message_start" else None
|
|
41
|
+
)
|
|
42
|
+
data = data or event.get("usage")
|
|
43
|
+
if isinstance(data, dict):
|
|
44
|
+
for field, attr in (
|
|
45
|
+
("input_tokens", "input_tokens"),
|
|
46
|
+
("cache_read_input_tokens", "cache_read_tokens"),
|
|
47
|
+
("cache_creation_input_tokens", "cache_write_tokens"),
|
|
48
|
+
("output_tokens", "output_tokens"),
|
|
49
|
+
):
|
|
50
|
+
if field in data:
|
|
51
|
+
setattr(usage, attr, _int(data[field]))
|
|
52
|
+
elif provider == "openai":
|
|
53
|
+
data = event.get("usage")
|
|
54
|
+
if isinstance(data, dict):
|
|
55
|
+
cached = _int((data.get("prompt_tokens_details") or {}).get("cached_tokens"))
|
|
56
|
+
usage.cache_read_tokens = cached
|
|
57
|
+
usage.input_tokens = max(0, _int(data.get("prompt_tokens")) - cached)
|
|
58
|
+
usage.output_tokens = _int(data.get("completion_tokens"))
|
|
59
|
+
elif provider == "gemini":
|
|
60
|
+
data = event.get("usageMetadata")
|
|
61
|
+
if isinstance(data, dict): # every chunk repeats the running totals
|
|
62
|
+
cached = _int(data.get("cachedContentTokenCount"))
|
|
63
|
+
usage.cache_read_tokens = cached
|
|
64
|
+
usage.input_tokens = max(0, _int(data.get("promptTokenCount")) - cached)
|
|
65
|
+
usage.output_tokens = _int(data.get("candidatesTokenCount")) + _int(
|
|
66
|
+
data.get("thoughtsTokenCount")
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class UsageSniffer:
|
|
71
|
+
def __init__(self, provider: str, content_type: str, content_encoding: str = ""):
|
|
72
|
+
self.provider = provider
|
|
73
|
+
self.usage = Usage()
|
|
74
|
+
self.found = False
|
|
75
|
+
self.streaming = "text/event-stream" in content_type
|
|
76
|
+
self.enabled = content_encoding in ("", "identity") # can't read compressed bytes
|
|
77
|
+
self._buffer = bytearray()
|
|
78
|
+
self._overflow = False
|
|
79
|
+
|
|
80
|
+
def feed(self, chunk: bytes) -> None:
|
|
81
|
+
if not self.enabled:
|
|
82
|
+
return
|
|
83
|
+
if self.streaming:
|
|
84
|
+
self._buffer.extend(chunk)
|
|
85
|
+
*lines, rest = bytes(self._buffer).split(b"\n")
|
|
86
|
+
self._buffer = bytearray(rest) # an incomplete last line waits for the next chunk
|
|
87
|
+
for line in lines:
|
|
88
|
+
self._line(line)
|
|
89
|
+
elif not self._overflow:
|
|
90
|
+
self._buffer.extend(chunk)
|
|
91
|
+
if len(self._buffer) > MAX_BUFFER_BYTES:
|
|
92
|
+
self._overflow = True
|
|
93
|
+
self._buffer = bytearray()
|
|
94
|
+
|
|
95
|
+
def _line(self, line: bytes) -> None:
|
|
96
|
+
line = line.strip()
|
|
97
|
+
if not line.startswith(b"data:"):
|
|
98
|
+
return
|
|
99
|
+
payload = line[len(b"data:") :].strip()
|
|
100
|
+
if not payload or payload == b"[DONE]":
|
|
101
|
+
return
|
|
102
|
+
self._apply(payload)
|
|
103
|
+
|
|
104
|
+
def _apply(self, payload: bytes) -> None:
|
|
105
|
+
try:
|
|
106
|
+
event = json.loads(payload)
|
|
107
|
+
except (ValueError, UnicodeDecodeError):
|
|
108
|
+
return
|
|
109
|
+
before = self.usage.as_dict()
|
|
110
|
+
apply_usage(self.provider, event, self.usage)
|
|
111
|
+
if self.usage.as_dict() != before:
|
|
112
|
+
self.found = True
|
|
113
|
+
|
|
114
|
+
def close(self) -> Usage | None:
|
|
115
|
+
if self.enabled and self.streaming and self._buffer:
|
|
116
|
+
self._line(bytes(self._buffer))
|
|
117
|
+
elif self.enabled and not self.streaming and not self._overflow and self._buffer:
|
|
118
|
+
self._apply(bytes(self._buffer))
|
|
119
|
+
self._buffer = bytearray()
|
|
120
|
+
return self.usage if self.found else None
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""Gateway-side tools: tools the gateway offers to the model and answers itself.
|
|
2
|
+
|
|
3
|
+
tokencut_retrieve return the exact original of an output TokenCut shortened (CCR)
|
|
4
|
+
tokencut_enable_tools switch on tool definitions that lazy tool loading hid
|
|
5
|
+
|
|
6
|
+
The agent does not know these tools. When the model calls one, the agent answers with an
|
|
7
|
+
error ("no such tool") in its next request; the gateway replaces that error with the real
|
|
8
|
+
answer before forwarding. The answer depends only on the call and the original content, so
|
|
9
|
+
it is the same on every later turn and the cached prefix stays byte-for-byte identical.
|
|
10
|
+
|
|
11
|
+
Lazy tool loading: every request carries only the core tools, the tools already used or
|
|
12
|
+
enabled in this conversation, and a short catalog of the rest (inside
|
|
13
|
+
tokencut_enable_tools). Decided from the request alone, so it survives gateway restarts.
|
|
14
|
+
|
|
15
|
+
Anthropic and OpenAI formats only (Gemini function calls have no ids to match answers).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from collections.abc import Iterator
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
|
|
22
|
+
from tokencut.gateway.ccr import RETRIEVE_TOOL, CCRStore, ref_for, slice_lines
|
|
23
|
+
from tokencut.gateway.history import DEFAULT_CORE_TOOLS, _forced_tool, _tool_lists, _tool_name
|
|
24
|
+
from tokencut.tokens import count_tokens
|
|
25
|
+
|
|
26
|
+
ENABLE_TOOL = "tokencut_enable_tools"
|
|
27
|
+
VIRTUAL_TOOLS = (RETRIEVE_TOOL, ENABLE_TOOL)
|
|
28
|
+
PROVIDERS = ("anthropic", "openai")
|
|
29
|
+
MIN_ORIGINAL_CHARS = 500 # smaller outputs are never shortened, so never retrieved
|
|
30
|
+
CATALOG_SUMMARY_CHARS = 90
|
|
31
|
+
CATALOG_MAX_CHARS = 4_000 # past this, hidden tools are listed by name only
|
|
32
|
+
# Requests with fewer client tools are side requests (Claude Code's WebSearch / summary
|
|
33
|
+
# calls): left alone. Provider server tools (web search, code execution) are never hidden.
|
|
34
|
+
MIN_CLIENT_TOOLS = 2
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class VirtualConfig:
|
|
40
|
+
ccr: bool = False # TOKENCUT_CCR: shortened outputs can be fetched back (tokencut_retrieve)
|
|
41
|
+
lazy_tools: bool = False # TOKENCUT_LAZY_TOOLS: only core / used tools + a catalog
|
|
42
|
+
core_tools: tuple[str, ...] = DEFAULT_CORE_TOOLS
|
|
43
|
+
store: CCRStore = field(default_factory=CCRStore)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
RETRIEVE_DESCRIPTION = (
|
|
47
|
+
"Return the exact original text of a tool output that TokenCut shortened (it ends with "
|
|
48
|
+
"'[TokenCut ref tc_...]'). Pass start_line / end_line (inclusive, numbered as in the "
|
|
49
|
+
"original output or in the omitted-lines note) to fetch only the part you need. Use this "
|
|
50
|
+
"instead of reading the file again or re-running the command."
|
|
51
|
+
)
|
|
52
|
+
RETRIEVE_SCHEMA = {
|
|
53
|
+
"type": "object",
|
|
54
|
+
"properties": {
|
|
55
|
+
"ref": {"type": "string", "description": "The reference, e.g. tc_1a2b3c4d5e6f"},
|
|
56
|
+
"start_line": {"type": "integer", "description": "First line to return (optional)"},
|
|
57
|
+
"end_line": {"type": "integer", "description": "Last line to return (optional)"},
|
|
58
|
+
},
|
|
59
|
+
"required": ["ref"],
|
|
60
|
+
}
|
|
61
|
+
ENABLE_SCHEMA = {
|
|
62
|
+
"type": "object",
|
|
63
|
+
"properties": {
|
|
64
|
+
"names": {"type": "array", "items": {"type": "string"},
|
|
65
|
+
"description": "Names of the tools to switch on"},
|
|
66
|
+
},
|
|
67
|
+
"required": ["names"],
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------- reading calls and answers from a raw body ----------
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _calls(provider: str, body: dict) -> Iterator[tuple[str, str, dict]]:
|
|
75
|
+
"""(call id, tool name, parsed input) for every tool call in the request."""
|
|
76
|
+
for message in body.get("messages", []):
|
|
77
|
+
if not isinstance(message, dict):
|
|
78
|
+
continue
|
|
79
|
+
if provider == "anthropic":
|
|
80
|
+
content = message.get("content")
|
|
81
|
+
for block in content if isinstance(content, list) else []:
|
|
82
|
+
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
83
|
+
data = block.get("input")
|
|
84
|
+
yield str(block.get("id", "")), str(block.get("name", "")), \
|
|
85
|
+
data if isinstance(data, dict) else {}
|
|
86
|
+
else:
|
|
87
|
+
for call in message.get("tool_calls") or []:
|
|
88
|
+
function = call.get("function", {}) if isinstance(call, dict) else {}
|
|
89
|
+
try:
|
|
90
|
+
data = json.loads(function.get("arguments") or "{}")
|
|
91
|
+
except (TypeError, ValueError):
|
|
92
|
+
data = {}
|
|
93
|
+
yield str(call.get("id", "")), str(function.get("name", "")), \
|
|
94
|
+
data if isinstance(data, dict) else {}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _result_texts(provider: str, body: dict) -> Iterator[str]:
|
|
98
|
+
"""Text of every tool result in the request."""
|
|
99
|
+
for message in body.get("messages", []):
|
|
100
|
+
if not isinstance(message, dict):
|
|
101
|
+
continue
|
|
102
|
+
if provider == "anthropic":
|
|
103
|
+
content = message.get("content")
|
|
104
|
+
for block in content if isinstance(content, list) else []:
|
|
105
|
+
if not isinstance(block, dict) or block.get("type") != "tool_result":
|
|
106
|
+
continue
|
|
107
|
+
inner = block.get("content")
|
|
108
|
+
if isinstance(inner, str):
|
|
109
|
+
yield inner
|
|
110
|
+
elif isinstance(inner, list):
|
|
111
|
+
for sub in inner:
|
|
112
|
+
if isinstance(sub, dict) and sub.get("type") == "text":
|
|
113
|
+
yield str(sub.get("text", ""))
|
|
114
|
+
elif message.get("role") in ("tool", "function") and isinstance(message.get("content"), str):
|
|
115
|
+
yield message["content"]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _set_answer(provider: str, body: dict, call_id: str, text: str) -> bool:
|
|
119
|
+
"""Replace the agent's answer to call `call_id` with `text`. False if there is none yet."""
|
|
120
|
+
for message in body.get("messages", []):
|
|
121
|
+
if not isinstance(message, dict):
|
|
122
|
+
continue
|
|
123
|
+
if provider == "anthropic":
|
|
124
|
+
content = message.get("content")
|
|
125
|
+
for block in content if isinstance(content, list) else []:
|
|
126
|
+
if (isinstance(block, dict) and block.get("type") == "tool_result"
|
|
127
|
+
and block.get("tool_use_id") == call_id):
|
|
128
|
+
block["content"] = text
|
|
129
|
+
block.pop("is_error", None)
|
|
130
|
+
return True
|
|
131
|
+
elif message.get("role") == "tool" and message.get("tool_call_id") == call_id:
|
|
132
|
+
message["content"] = text
|
|
133
|
+
return True
|
|
134
|
+
return False
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ---------- answering the gateway's own tools ----------
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _retrieve_answer(data: dict, originals: dict[str, str], store: CCRStore | None) -> str:
|
|
141
|
+
ref = str(data.get("ref", "")).strip()
|
|
142
|
+
original = originals.get(ref) or (store.get(ref) if store is not None else None)
|
|
143
|
+
if original is None:
|
|
144
|
+
return (f"[TokenCut: {ref or 'this reference'} is no longer available; read the file "
|
|
145
|
+
"again or re-run the command]")
|
|
146
|
+
|
|
147
|
+
def as_int(key: str) -> int | None:
|
|
148
|
+
try:
|
|
149
|
+
return int(data[key]) if data.get(key) is not None else None
|
|
150
|
+
except (TypeError, ValueError):
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
text, lo, hi = slice_lines(original, as_int("start_line"), as_int("end_line"))
|
|
154
|
+
if not text:
|
|
155
|
+
return f"[TokenCut: {ref} has no lines in that range]"
|
|
156
|
+
return f"[TokenCut: original of {ref}, lines {lo}-{hi}]\n{text}"
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _enable_answer(data: dict, known: set[str]) -> str:
|
|
160
|
+
names = [str(n) for n in data.get("names") or [] if isinstance(n, str | int)]
|
|
161
|
+
found = [n for n in names if n in known]
|
|
162
|
+
missing = [n for n in names if n not in known]
|
|
163
|
+
parts = []
|
|
164
|
+
if found:
|
|
165
|
+
parts.append(f"enabled {', '.join(found)}; call them directly from now on")
|
|
166
|
+
if missing:
|
|
167
|
+
parts.append(f"no tool named {', '.join(missing)}")
|
|
168
|
+
return f"[TokenCut: {'; '.join(parts) or 'no tools named'}]"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def resolve(provider: str, body: dict, store: CCRStore | None) -> list[dict]:
|
|
172
|
+
"""Answer the gateway's own tool calls in place. Returns report entries."""
|
|
173
|
+
if provider not in PROVIDERS:
|
|
174
|
+
return []
|
|
175
|
+
calls = [(cid, name, data) for cid, name, data in _calls(provider, body) if name in VIRTUAL_TOOLS]
|
|
176
|
+
if not calls:
|
|
177
|
+
return []
|
|
178
|
+
originals: dict[str, str] = {}
|
|
179
|
+
if any(name == RETRIEVE_TOOL for _, name, _ in calls):
|
|
180
|
+
for text in _result_texts(provider, body):
|
|
181
|
+
if len(text) >= MIN_ORIGINAL_CHARS:
|
|
182
|
+
originals.setdefault(ref_for(text), text)
|
|
183
|
+
known = {_tool_name(t, how) for tools, how in _tool_lists(body) for t in tools}
|
|
184
|
+
entries = []
|
|
185
|
+
for call_id, name, data in calls:
|
|
186
|
+
answer = (_retrieve_answer(data, originals, store) if name == RETRIEVE_TOOL
|
|
187
|
+
else _enable_answer(data, known))
|
|
188
|
+
if _set_answer(provider, body, call_id, answer):
|
|
189
|
+
how = "ccr_retrieved" if name == RETRIEVE_TOOL else "tools_enabled"
|
|
190
|
+
entries.append({"segment": call_id, "how": how, "tool": name,
|
|
191
|
+
"before": 0, "after": count_tokens(answer)})
|
|
192
|
+
return entries
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ---------- tool definitions sent upstream ----------
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _summary(tool: dict, how: str) -> str:
|
|
199
|
+
desc = tool.get("description") or (tool.get("function") or {}).get("description") or ""
|
|
200
|
+
first = str(desc).strip().split("\n")[0].split(". ")[0]
|
|
201
|
+
return first[:CATALOG_SUMMARY_CHARS]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _definition(provider: str, name: str, description: str, schema: dict) -> dict:
|
|
205
|
+
if provider == "openai":
|
|
206
|
+
return {"type": "function",
|
|
207
|
+
"function": {"name": name, "description": description, "parameters": schema}}
|
|
208
|
+
return {"name": name, "description": description, "input_schema": schema}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _catalog(hidden: list[tuple[str, str]]) -> str:
|
|
212
|
+
head = ("Switch on tools that are hidden to keep requests small. Call this with their names "
|
|
213
|
+
"when you need one; afterwards call them directly. Hidden tools:")
|
|
214
|
+
lines, used = [], len(head)
|
|
215
|
+
for i, (name, summary) in enumerate(hidden):
|
|
216
|
+
line = f"\n- {name}: {summary}" if summary else f"\n- {name}"
|
|
217
|
+
if used + len(line) > CATALOG_MAX_CHARS:
|
|
218
|
+
lines.append("\n- also: " + ", ".join(n for n, _ in hidden[i:]))
|
|
219
|
+
break
|
|
220
|
+
lines.append(line)
|
|
221
|
+
used += len(line)
|
|
222
|
+
return head + "".join(lines)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def finish_tools(provider: str, body: dict, ccr: bool, lazy: bool,
|
|
226
|
+
core_tools: tuple[str, ...]) -> list[dict]:
|
|
227
|
+
"""Lazy-prune the tool list and add the gateway's tools, in place. Report entries."""
|
|
228
|
+
if not (ccr or lazy) or not offers_tools(provider, body):
|
|
229
|
+
return [] # no tools (plain chat), a side request, or a legacy "functions" list
|
|
230
|
+
tools, how = _tool_lists(body)[0]
|
|
231
|
+
entries = []
|
|
232
|
+
if lazy:
|
|
233
|
+
called = {name for _, name, _ in _calls(provider, body)}
|
|
234
|
+
enabled = {str(n) for _, name, data in _calls(provider, body) if name == ENABLE_TOOL
|
|
235
|
+
for n in data.get("names") or []}
|
|
236
|
+
keep = set(core_tools) | called | enabled | {_forced_tool(body)}
|
|
237
|
+
kept, hidden, removed, cache_mark = [], [], 0, None
|
|
238
|
+
for tool in tools:
|
|
239
|
+
name = _tool_name(tool, how)
|
|
240
|
+
if not name or name in keep or name in VIRTUAL_TOOLS or _is_server_tool(tool):
|
|
241
|
+
kept.append(tool)
|
|
242
|
+
continue
|
|
243
|
+
hidden.append((name, _summary(tool, how)))
|
|
244
|
+
removed += count_tokens(json.dumps(tool, ensure_ascii=False, sort_keys=True))
|
|
245
|
+
cache_mark = tool.get("cache_control", cache_mark)
|
|
246
|
+
if hidden:
|
|
247
|
+
# Keep the provider's cache breakpoint if it sat on a hidden tool.
|
|
248
|
+
if cache_mark is not None and kept and not any("cache_control" in t for t in kept):
|
|
249
|
+
kept[-1] = {**kept[-1], "cache_control": cache_mark}
|
|
250
|
+
tools[:] = kept
|
|
251
|
+
catalog = _definition(provider, ENABLE_TOOL, _catalog(hidden), ENABLE_SCHEMA)
|
|
252
|
+
tools.append(catalog)
|
|
253
|
+
entries.append({"segment": "tools", "how": "tools_lazy", "tool": "",
|
|
254
|
+
"before": removed, "after": count_tokens(json.dumps(catalog))})
|
|
255
|
+
if ccr:
|
|
256
|
+
retrieve = _definition(provider, RETRIEVE_TOOL, RETRIEVE_DESCRIPTION, RETRIEVE_SCHEMA)
|
|
257
|
+
tools.append(retrieve)
|
|
258
|
+
entries.append({"segment": "tools", "how": "ccr_tool", "tool": "",
|
|
259
|
+
"before": 0, "after": count_tokens(json.dumps(retrieve))})
|
|
260
|
+
return entries
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def offers_tools(provider: str, body: dict) -> bool:
|
|
264
|
+
"""Will finish_tools be able to add the gateway's tools to this request?"""
|
|
265
|
+
lists = _tool_lists(body)
|
|
266
|
+
if provider not in PROVIDERS or len(lists) != 1 or lists[0][0] is not body.get("tools"):
|
|
267
|
+
return False
|
|
268
|
+
return sum(1 for t in lists[0][0] if not _is_server_tool(t)) >= MIN_CLIENT_TOOLS
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _is_server_tool(tool: dict) -> bool:
|
|
272
|
+
"""A tool the provider runs itself (Anthropic web_search_..., code_execution_...)."""
|
|
273
|
+
return tool.get("type") not in (None, "custom", "function")
|
|
File without changes
|