tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""OpenAI Chat Completions API (POST /v1/chat/completions).
|
|
2
|
+
|
|
3
|
+
Also the format of most "OpenAI-compatible" endpoints: OpenRouter, Cline, Cursor custom
|
|
4
|
+
models, local servers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
8
|
+
from tokencut.gateway.adapters.common import SegmentBuilder, as_json
|
|
9
|
+
|
|
10
|
+
PROVIDER = "openai"
|
|
11
|
+
_SYSTEM_ROLES = ("system", "developer")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def model_of(body: dict, path: str) -> str:
|
|
15
|
+
return str(body.get("model", ""))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def parse(body: dict) -> list[Segment]:
|
|
19
|
+
b = SegmentBuilder()
|
|
20
|
+
tools = body.get("tools") or body.get("functions")
|
|
21
|
+
if tools:
|
|
22
|
+
b.add(
|
|
23
|
+
"tools",
|
|
24
|
+
SegmentKind.TOOLS,
|
|
25
|
+
"system",
|
|
26
|
+
-1,
|
|
27
|
+
["tools" if "tools" in body else "functions"],
|
|
28
|
+
as_json(tools),
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
tool_names: dict[str, str] = {} # tool_call id -> function name
|
|
32
|
+
for i, message in enumerate(body.get("messages", [])):
|
|
33
|
+
role = str(message.get("role", ""))
|
|
34
|
+
if role in _SYSTEM_ROLES:
|
|
35
|
+
kind = SegmentKind.SYSTEM
|
|
36
|
+
elif role == "assistant":
|
|
37
|
+
kind = SegmentKind.ASSISTANT_TEXT
|
|
38
|
+
else:
|
|
39
|
+
kind = SegmentKind.USER_TEXT
|
|
40
|
+
is_result = role in ("tool", "function")
|
|
41
|
+
call_id = str(message.get("tool_call_id", ""))
|
|
42
|
+
name = tool_names.get(call_id, str(message.get("name", "")))
|
|
43
|
+
|
|
44
|
+
content = message.get("content")
|
|
45
|
+
parts: list[tuple[str, list[str | int], str]] = [] # (id, path, text)
|
|
46
|
+
if isinstance(content, str):
|
|
47
|
+
parts.append((f"m{i}", ["messages", i, "content"], content))
|
|
48
|
+
elif isinstance(content, list):
|
|
49
|
+
for k, part in enumerate(content):
|
|
50
|
+
if isinstance(part, dict) and part.get("type") == "text":
|
|
51
|
+
parts.append(
|
|
52
|
+
(
|
|
53
|
+
f"m{i}.p{k}",
|
|
54
|
+
["messages", i, "content", k, "text"],
|
|
55
|
+
str(part.get("text", "")),
|
|
56
|
+
)
|
|
57
|
+
)
|
|
58
|
+
else: # image_url, input_audio, file
|
|
59
|
+
b.add(
|
|
60
|
+
f"m{i}.p{k}", SegmentKind.MEDIA, role, i, ["messages", i, "content", k], ""
|
|
61
|
+
)
|
|
62
|
+
for seg_id, path, text in parts:
|
|
63
|
+
if is_result:
|
|
64
|
+
b.add_tool_result(seg_id, role, i, path, text, name, call_id)
|
|
65
|
+
else:
|
|
66
|
+
b.add(seg_id, kind, role, i, path, text)
|
|
67
|
+
|
|
68
|
+
for j, call in enumerate(message.get("tool_calls") or []):
|
|
69
|
+
function = call.get("function", {}) if isinstance(call, dict) else {}
|
|
70
|
+
call_name = str(function.get("name", ""))
|
|
71
|
+
tool_names[str(call.get("id", ""))] = call_name
|
|
72
|
+
b.add(
|
|
73
|
+
f"m{i}.t{j}",
|
|
74
|
+
SegmentKind.TOOL_CALL,
|
|
75
|
+
role,
|
|
76
|
+
i,
|
|
77
|
+
["messages", i, "tool_calls", j, "function", "arguments"],
|
|
78
|
+
str(function.get("arguments", "")),
|
|
79
|
+
call_name,
|
|
80
|
+
str(call.get("id", "")),
|
|
81
|
+
)
|
|
82
|
+
return b.segments
|
tokencut/gateway/app.py
ADDED
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"""TokenCut gateway (Iteration 2).
|
|
2
|
+
|
|
3
|
+
Sits between a coding agent and its model provider. Each request is split into
|
|
4
|
+
segments and measured, forwarded, and the response is streamed straight back while the
|
|
5
|
+
provider's billed usage is read from it. Two modes (TOKENCUT_MODE):
|
|
6
|
+
shadow (default) forward unchanged; report what trimming would save
|
|
7
|
+
trim send the cache-safe trimmed request (see trimmer.py); fail open
|
|
8
|
+
|
|
9
|
+
Point an agent at it:
|
|
10
|
+
Claude Code: ANTHROPIC_BASE_URL=http://127.0.0.1:8787/anthropic
|
|
11
|
+
OpenAI SDKs: base_url = http://127.0.0.1:8787/openai/v1
|
|
12
|
+
Gemini: base URL http://127.0.0.1:8787/gemini
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import hmac
|
|
17
|
+
import json
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import time
|
|
22
|
+
from contextlib import asynccontextmanager
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import httpx2
|
|
27
|
+
from fastapi import FastAPI, Request, Response
|
|
28
|
+
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
|
29
|
+
|
|
30
|
+
from tokencut.contracts import Tier
|
|
31
|
+
from tokencut.gateway.history import CacheEventConfig
|
|
32
|
+
from tokencut.gateway.records import RequestLog, analyse, now_iso
|
|
33
|
+
from tokencut.gateway.routing import (
|
|
34
|
+
RouteConfig,
|
|
35
|
+
claude_litellm_headers,
|
|
36
|
+
claude_via_litellm,
|
|
37
|
+
gpt_headers,
|
|
38
|
+
model_from_body,
|
|
39
|
+
move_tool_documents,
|
|
40
|
+
pick_route,
|
|
41
|
+
)
|
|
42
|
+
from tokencut.gateway.session import SessionStore
|
|
43
|
+
from tokencut.gateway.task_router import load_router
|
|
44
|
+
from tokencut.gateway.trimmer import TrimConfig
|
|
45
|
+
from tokencut.gateway.usage import UsageSniffer
|
|
46
|
+
from tokencut.gateway.virtual import VirtualConfig
|
|
47
|
+
|
|
48
|
+
log = logging.getLogger("tokencut.gateway")
|
|
49
|
+
_LIVE_PAGE = Path(__file__).with_name("live.html")
|
|
50
|
+
|
|
51
|
+
DEFAULT_UPSTREAMS = {
|
|
52
|
+
"anthropic": "https://api.anthropic.com",
|
|
53
|
+
"openai": "https://api.openai.com",
|
|
54
|
+
"gemini": "https://generativelanguage.googleapis.com",
|
|
55
|
+
}
|
|
56
|
+
# Requests we measure; every other path (model lists, token counting, ...) is only forwarded.
|
|
57
|
+
_INFERENCE_PATHS = {
|
|
58
|
+
"anthropic": re.compile(r"(^|/)v1/messages$"),
|
|
59
|
+
"openai": re.compile(r"(^|/)chat/completions$"),
|
|
60
|
+
"gemini": re.compile(r"models/[^/:]+:(generateContent|streamGenerateContent)$"),
|
|
61
|
+
}
|
|
62
|
+
# Hop-by-hop / transport headers that must not be copied between the two connections.
|
|
63
|
+
# accept-encoding is dropped so the provider answers uncompressed and usage can be read.
|
|
64
|
+
_DROP_REQUEST_HEADERS = {
|
|
65
|
+
"host",
|
|
66
|
+
"content-length",
|
|
67
|
+
"connection",
|
|
68
|
+
"keep-alive",
|
|
69
|
+
"transfer-encoding",
|
|
70
|
+
"te",
|
|
71
|
+
"upgrade",
|
|
72
|
+
"proxy-authorization",
|
|
73
|
+
"proxy-connection",
|
|
74
|
+
"accept-encoding",
|
|
75
|
+
"x-tokencut-key", # the local client key never leaves the gateway
|
|
76
|
+
}
|
|
77
|
+
_DROP_RESPONSE_HEADERS = {"content-length", "connection", "keep-alive", "transfer-encoding"}
|
|
78
|
+
_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class GatewayConfig:
|
|
83
|
+
upstreams: dict[str, str] = field(default_factory=lambda: dict(DEFAULT_UPSTREAMS))
|
|
84
|
+
log_dir: Path | None = Path("logs/gateway")
|
|
85
|
+
capture: bool = False # save full request bodies (local only) for test fixtures
|
|
86
|
+
mode: str = "shadow" # "shadow": forward unchanged, report possible savings; "trim": apply
|
|
87
|
+
trim: TrimConfig = field(default_factory=TrimConfig)
|
|
88
|
+
cache_events: bool = True # re-trim history when the provider cache is lost anyway
|
|
89
|
+
events: CacheEventConfig = field(default_factory=CacheEventConfig)
|
|
90
|
+
virtual: VirtualConfig = field(default_factory=VirtualConfig) # CCR, lazy tool loading
|
|
91
|
+
route: RouteConfig = field(default_factory=RouteConfig) # dual-plan demo (GPT via LiteLLM)
|
|
92
|
+
router_dir: str | None = None # prompt router model folder (TOKENCUT_ROUTER_DIR)
|
|
93
|
+
router_repo: str | None = None # else this public Hugging Face repo (TOKENCUT_ROUTER_REPO)
|
|
94
|
+
router: object | None = None # loaded router (tests pass a fake)
|
|
95
|
+
client_key: str | None = None # TOKENCUT_KEY: proxy requests must send it in x-tokencut-key
|
|
96
|
+
|
|
97
|
+
@classmethod
|
|
98
|
+
def from_env(cls) -> "GatewayConfig":
|
|
99
|
+
upstreams = {
|
|
100
|
+
name: os.environ.get(f"TOKENCUT_{name.upper()}_UPSTREAM", url).rstrip("/")
|
|
101
|
+
for name, url in DEFAULT_UPSTREAMS.items()
|
|
102
|
+
}
|
|
103
|
+
log_dir = os.environ.get("TOKENCUT_LOG_DIR", "logs/gateway")
|
|
104
|
+
mode = os.environ.get("TOKENCUT_MODE", "shadow")
|
|
105
|
+
if mode not in ("shadow", "trim"):
|
|
106
|
+
raise ValueError(f"TOKENCUT_MODE must be 'shadow' or 'trim', not {mode!r}")
|
|
107
|
+
trim = TrimConfig(
|
|
108
|
+
tier=Tier(os.environ.get("TOKENCUT_TIER", TrimConfig.tier.value)),
|
|
109
|
+
min_tokens=int(os.environ.get("TOKENCUT_MIN_TRIM_TOKENS", TrimConfig.min_tokens)),
|
|
110
|
+
resize_images=os.environ.get("TOKENCUT_RESIZE_IMAGES", "1") != "0",
|
|
111
|
+
trim_borders=os.environ.get("TOKENCUT_TRIM_BORDERS", "1") != "0",
|
|
112
|
+
media_user=os.environ.get("TOKENCUT_MEDIA_USER", "") == "1",
|
|
113
|
+
pdf_text_pages=os.environ.get("TOKENCUT_PDF_TEXT_PAGES", "1") != "0",
|
|
114
|
+
system_note=os.environ.get("TOKENCUT_SYSTEM_NOTE", "") == "1",
|
|
115
|
+
prompt_brief=os.environ.get("TOKENCUT_PROMPT_BRIEF", "") == "1",
|
|
116
|
+
brief_threshold=float(os.environ.get("TOKENCUT_BRIEF_THRESHOLD",
|
|
117
|
+
TrimConfig.brief_threshold)),
|
|
118
|
+
)
|
|
119
|
+
events = CacheEventConfig(
|
|
120
|
+
cache_ttl_seconds=int(os.environ.get("TOKENCUT_CACHE_TTL",
|
|
121
|
+
CacheEventConfig.cache_ttl_seconds)),
|
|
122
|
+
prune_tools=os.environ.get("TOKENCUT_PRUNE_TOOLS", "") == "1",
|
|
123
|
+
task_masking=os.environ.get("TOKENCUT_TASK_MASK", "1") != "0",
|
|
124
|
+
task_mask_min_tokens=int(os.environ.get("TOKENCUT_TASK_MASK_TOKENS",
|
|
125
|
+
CacheEventConfig.task_mask_min_tokens)),
|
|
126
|
+
)
|
|
127
|
+
core = os.environ.get("TOKENCUT_CORE_TOOLS")
|
|
128
|
+
if core:
|
|
129
|
+
events.core_tools = tuple(t.strip() for t in core.split(",") if t.strip())
|
|
130
|
+
virtual = VirtualConfig(
|
|
131
|
+
ccr=os.environ.get("TOKENCUT_CCR", "") == "1",
|
|
132
|
+
lazy_tools=os.environ.get("TOKENCUT_LAZY_TOOLS", "") == "1",
|
|
133
|
+
core_tools=events.core_tools,
|
|
134
|
+
)
|
|
135
|
+
return cls(
|
|
136
|
+
upstreams=upstreams,
|
|
137
|
+
log_dir=Path(log_dir) if log_dir else None,
|
|
138
|
+
capture=os.environ.get("TOKENCUT_CAPTURE", "") == "1",
|
|
139
|
+
mode=mode,
|
|
140
|
+
trim=trim,
|
|
141
|
+
cache_events=os.environ.get("TOKENCUT_CACHE_EVENTS", "1") != "0",
|
|
142
|
+
events=events,
|
|
143
|
+
virtual=virtual,
|
|
144
|
+
route=RouteConfig.from_env(),
|
|
145
|
+
router_dir=os.environ.get("TOKENCUT_ROUTER_DIR") or None,
|
|
146
|
+
router_repo=os.environ.get("TOKENCUT_ROUTER_REPO") or None,
|
|
147
|
+
client_key=os.environ.get("TOKENCUT_KEY") or None,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def create_app(
|
|
152
|
+
config: GatewayConfig | None = None, transport: httpx2.AsyncBaseTransport | None = None
|
|
153
|
+
) -> FastAPI:
|
|
154
|
+
config = config or GatewayConfig.from_env()
|
|
155
|
+
request_log = RequestLog(config.log_dir, config.capture)
|
|
156
|
+
# Conversation memory for cache-event trimming (in shadow mode its decisions are never sent).
|
|
157
|
+
session = (SessionStore(), config.events) if config.cache_events else None
|
|
158
|
+
if config.trim.prompt_brief and config.router is None: # once, at start; None = no briefs
|
|
159
|
+
config.router = load_router(config.router_dir, config.router_repo)
|
|
160
|
+
timeout = httpx2.Timeout(connect=10.0, read=600.0, write=60.0, pool=10.0)
|
|
161
|
+
|
|
162
|
+
@asynccontextmanager
|
|
163
|
+
async def lifespan(app: FastAPI):
|
|
164
|
+
app.state.client = httpx2.AsyncClient(timeout=timeout, transport=transport)
|
|
165
|
+
yield
|
|
166
|
+
await app.state.client.aclose()
|
|
167
|
+
|
|
168
|
+
app = FastAPI(title="TokenCut gateway", version="0.2.0", lifespan=lifespan)
|
|
169
|
+
app.state.request_log = request_log
|
|
170
|
+
|
|
171
|
+
@app.get("/tokencut/health")
|
|
172
|
+
async def health() -> dict:
|
|
173
|
+
return {
|
|
174
|
+
"status": "ok",
|
|
175
|
+
"mode": config.mode,
|
|
176
|
+
"tier": config.trim.tier.value,
|
|
177
|
+
"cache_events": config.cache_events,
|
|
178
|
+
"prune_tools": config.events.prune_tools,
|
|
179
|
+
"task_masking": config.events.task_masking,
|
|
180
|
+
"ccr": config.virtual.ccr,
|
|
181
|
+
"lazy_tools": config.virtual.lazy_tools,
|
|
182
|
+
"gpt_upstream": config.route.gpt_upstream,
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
@app.get("/tokencut/stats")
|
|
186
|
+
async def stats() -> dict:
|
|
187
|
+
return request_log.summary()
|
|
188
|
+
|
|
189
|
+
@app.get("/tokencut/recent")
|
|
190
|
+
async def recent() -> list[dict]:
|
|
191
|
+
return request_log.recent_feed()
|
|
192
|
+
|
|
193
|
+
@app.get("/tokencut/", response_class=HTMLResponse)
|
|
194
|
+
async def live_page() -> str:
|
|
195
|
+
return _LIVE_PAGE.read_text(encoding="utf-8")
|
|
196
|
+
|
|
197
|
+
async def forward(provider: str, path: str, request: Request) -> Response:
|
|
198
|
+
if config.client_key and not hmac.compare_digest(
|
|
199
|
+
request.headers.get("x-tokencut-key", "").encode(), config.client_key.encode()
|
|
200
|
+
):
|
|
201
|
+
# Only local programs given the key may use the gateway (and the user's logins).
|
|
202
|
+
return JSONResponse(status_code=401, content={"type": "error", "error": {
|
|
203
|
+
"type": "authentication_error",
|
|
204
|
+
"message": "TokenCut: missing or wrong x-tokencut-key (start the agent with "
|
|
205
|
+
"`tokencut`)"}})
|
|
206
|
+
started = time.perf_counter()
|
|
207
|
+
body = await request.body()
|
|
208
|
+
record: dict = {
|
|
209
|
+
"ts": now_iso(),
|
|
210
|
+
"provider": provider,
|
|
211
|
+
"method": request.method,
|
|
212
|
+
"path": path,
|
|
213
|
+
"analysis": None,
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
analysis_task = None
|
|
217
|
+
send_body = body
|
|
218
|
+
if request.method == "POST" and _INFERENCE_PATHS[provider].search(path):
|
|
219
|
+
request_log.save_capture(provider, body)
|
|
220
|
+
if config.mode == "trim":
|
|
221
|
+
# Trim before sending. Any failure: send the original request (fail open).
|
|
222
|
+
try:
|
|
223
|
+
analysis, trimmed = await asyncio.to_thread(
|
|
224
|
+
analyse, provider, path, body, config.trim, session, None, config.virtual,
|
|
225
|
+
config.router
|
|
226
|
+
)
|
|
227
|
+
record["analysis"] = analysis
|
|
228
|
+
if trimmed is not None:
|
|
229
|
+
send_body = trimmed
|
|
230
|
+
record["trim_applied"] = True
|
|
231
|
+
except Exception as exc:
|
|
232
|
+
log.warning("trimming failed; forwarding original", exc_info=True)
|
|
233
|
+
record["analysis_error"] = f"{type(exc).__name__}: {exc}"[:300]
|
|
234
|
+
else:
|
|
235
|
+
# Shadow: measure (and work out the possible trim) in a worker thread while the
|
|
236
|
+
# unchanged request is already on its way, so no latency is added.
|
|
237
|
+
analysis_task = asyncio.create_task(
|
|
238
|
+
asyncio.to_thread(
|
|
239
|
+
analyse, provider, path, body, config.trim, session, None, config.virtual,
|
|
240
|
+
config.router
|
|
241
|
+
)
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
headers = {
|
|
245
|
+
k: v for k, v in request.headers.items() if k.lower() not in _DROP_REQUEST_HEADERS
|
|
246
|
+
}
|
|
247
|
+
route = pick_route(provider, model_from_body(body), config.route)
|
|
248
|
+
record["route"] = route
|
|
249
|
+
query = f"?{request.url.query}" if request.url.query else ""
|
|
250
|
+
direct_url = f"{config.upstreams[provider]}/{path}{query}"
|
|
251
|
+
direct_headers = headers
|
|
252
|
+
is_inference = _INFERENCE_PATHS[provider].search(path) is not None
|
|
253
|
+
if route == "gpt":
|
|
254
|
+
# Same Anthropic-format body; LiteLLM translates it for the ChatGPT plan.
|
|
255
|
+
url = f"{config.route.gpt_upstream}/{path}{query}"
|
|
256
|
+
headers = gpt_headers(headers, config.route)
|
|
257
|
+
record["via"] = "litellm"
|
|
258
|
+
if is_inference:
|
|
259
|
+
# ChatGPT rejects PDFs inside tool results: move them next to the tool result
|
|
260
|
+
# (in shadow mode too, or the request fails). Fail open: send as it was.
|
|
261
|
+
try:
|
|
262
|
+
moved_body, moved = move_tool_documents(json.loads(send_body))
|
|
263
|
+
if moved:
|
|
264
|
+
send_body = json.dumps(moved_body, ensure_ascii=False,
|
|
265
|
+
separators=(",", ":")).encode()
|
|
266
|
+
record["gpt_documents_moved"] = moved
|
|
267
|
+
except (ValueError, TypeError, AttributeError):
|
|
268
|
+
log.warning("could not move tool documents for the GPT route", exc_info=True)
|
|
269
|
+
elif route == "claude" and is_inference and claude_via_litellm(config.route):
|
|
270
|
+
# LiteLLM forwards the Claude plan login to Anthropic; direct send is the fallback.
|
|
271
|
+
url = f"{config.route.gpt_upstream}/{path}{query}"
|
|
272
|
+
headers = claude_litellm_headers(headers, config.route)
|
|
273
|
+
record["via"] = "litellm"
|
|
274
|
+
else:
|
|
275
|
+
url = direct_url
|
|
276
|
+
if route in ("claude", "gpt"):
|
|
277
|
+
record["via"] = "direct"
|
|
278
|
+
# Ask for an uncompressed reply, or usage can't be read (the client's default is gzip).
|
|
279
|
+
headers["accept-encoding"] = "identity"
|
|
280
|
+
client: httpx2.AsyncClient = app.state.client
|
|
281
|
+
try:
|
|
282
|
+
try:
|
|
283
|
+
upstream = await client.send(
|
|
284
|
+
client.build_request(request.method, url, headers=headers, content=send_body),
|
|
285
|
+
stream=True,
|
|
286
|
+
)
|
|
287
|
+
except httpx2.RequestError:
|
|
288
|
+
if not (route == "claude" and record.get("via") == "litellm"):
|
|
289
|
+
raise
|
|
290
|
+
# LiteLLM down: send the Claude request straight to Anthropic (fail open).
|
|
291
|
+
log.warning("LiteLLM unreachable; sending Claude request directly")
|
|
292
|
+
record.update(via="direct", fallback="litellm_unreachable")
|
|
293
|
+
upstream = await client.send(
|
|
294
|
+
client.build_request(
|
|
295
|
+
request.method, direct_url, headers=direct_headers, content=send_body
|
|
296
|
+
),
|
|
297
|
+
stream=True,
|
|
298
|
+
)
|
|
299
|
+
except httpx2.RequestError as exc:
|
|
300
|
+
record.update(status=502, error=f"upstream unreachable: {type(exc).__name__}")
|
|
301
|
+
await _finish(record, analysis_task, None, started, request_log)
|
|
302
|
+
return JSONResponse(
|
|
303
|
+
status_code=502,
|
|
304
|
+
content={
|
|
305
|
+
"error": {
|
|
306
|
+
"type": "tokencut_upstream_error",
|
|
307
|
+
"message": f"TokenCut could not reach {provider}: {exc}",
|
|
308
|
+
}
|
|
309
|
+
},
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
record["status"] = upstream.status_code
|
|
313
|
+
sniffer = UsageSniffer(
|
|
314
|
+
provider,
|
|
315
|
+
upstream.headers.get("content-type", ""),
|
|
316
|
+
upstream.headers.get("content-encoding", ""),
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
async def relay():
|
|
320
|
+
try:
|
|
321
|
+
async for chunk in upstream.aiter_raw():
|
|
322
|
+
sniffer.feed(chunk)
|
|
323
|
+
yield chunk
|
|
324
|
+
finally:
|
|
325
|
+
await upstream.aclose()
|
|
326
|
+
await _finish(record, analysis_task, sniffer, started, request_log)
|
|
327
|
+
|
|
328
|
+
response_headers = {
|
|
329
|
+
k: v for k, v in upstream.headers.items() if k.lower() not in _DROP_RESPONSE_HEADERS
|
|
330
|
+
}
|
|
331
|
+
return StreamingResponse(
|
|
332
|
+
relay(), status_code=upstream.status_code, headers=response_headers
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
for provider in DEFAULT_UPSTREAMS:
|
|
336
|
+
|
|
337
|
+
async def route(path: str, request: Request, provider: str = provider) -> Response:
|
|
338
|
+
return await forward(provider, path, request)
|
|
339
|
+
|
|
340
|
+
app.add_api_route(
|
|
341
|
+
f"/{provider}/{{path:path}}", route, methods=_METHODS, include_in_schema=False
|
|
342
|
+
)
|
|
343
|
+
return app
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
async def _finish(
|
|
347
|
+
record: dict,
|
|
348
|
+
analysis_task,
|
|
349
|
+
sniffer: UsageSniffer | None,
|
|
350
|
+
started: float,
|
|
351
|
+
request_log: RequestLog,
|
|
352
|
+
) -> None:
|
|
353
|
+
"""Complete and store the record. Never raises: measuring must not break forwarding."""
|
|
354
|
+
try:
|
|
355
|
+
record["latency_ms"] = round((time.perf_counter() - started) * 1000, 1)
|
|
356
|
+
if analysis_task is not None:
|
|
357
|
+
try:
|
|
358
|
+
record["analysis"], _ = await analysis_task
|
|
359
|
+
except Exception as exc: # bad JSON, unexpected shape: forwarded anyway
|
|
360
|
+
log.warning("could not analyse %s request", record["provider"], exc_info=True)
|
|
361
|
+
record["analysis_error"] = f"{type(exc).__name__}: {exc}"[:300]
|
|
362
|
+
usage = sniffer.close() if sniffer is not None else None
|
|
363
|
+
record["usage"] = usage.as_dict() if usage is not None else None
|
|
364
|
+
request_log.add(record)
|
|
365
|
+
except Exception:
|
|
366
|
+
log.exception("could not record request")
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Task briefs: a short, task-specific note appended to a typed user message (prompt enhancement).
|
|
2
|
+
|
|
3
|
+
For every typed user message in the request, oldest first: rules and the router
|
|
4
|
+
(task_router.py) give a task type; a brief is added only when the type is a real task, the
|
|
5
|
+
router is confident (p >= threshold, chosen on dev: RESULTS_REPORT §11) and the type differs from
|
|
6
|
+
the last brief given earlier in the same request (no repeats inside a task). The decision depends
|
|
7
|
+
only on the request, so the same messages get the same briefs on every turn: earlier messages
|
|
8
|
+
stay byte-identical (prompt cache) and nothing is remembered between requests.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
12
|
+
from tokencut.gateway.note import _is_agent_request
|
|
13
|
+
from tokencut.gateway.task_router import NO_BRIEF, prepare
|
|
14
|
+
from tokencut.tokens import count_tokens
|
|
15
|
+
|
|
16
|
+
TAG = "<tokencut_task_brief"
|
|
17
|
+
BRIEFS = {
|
|
18
|
+
"bugfix": ("Reproduce the problem first (a failing test or a command that shows it). Find the "
|
|
19
|
+
"root cause before editing; make the smallest fix; do not change existing tests to "
|
|
20
|
+
"make them pass. Run the failing test and the related tests and report the result."),
|
|
21
|
+
"feature": ("Follow how similar code in this repository is structured. Implement the smallest "
|
|
22
|
+
"version that meets the request, add or update tests for the new behaviour, run "
|
|
23
|
+
"them, and report what changed."),
|
|
24
|
+
"refactor": ("Behaviour must stay the same. Run the relevant tests before and after; change "
|
|
25
|
+
"structure only, no unrelated edits; keep public interfaces unless asked."),
|
|
26
|
+
"tests": ("Change tests only unless a code bug is confirmed. A new test must fail without the "
|
|
27
|
+
"fix and pass with it; never weaken existing assertions. Run the tests and report."),
|
|
28
|
+
"explain": ("Do not edit files. Read only what is needed (line ranges, not whole files); "
|
|
29
|
+
"answer with file paths and line numbers; keep it short."),
|
|
30
|
+
"run_action": ("Run what was asked and keep output short (e.g. -q); do not change code unless "
|
|
31
|
+
"asked. For git, check the status first; never force-push or rewrite history "
|
|
32
|
+
"unless asked."),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def brief_text(label: str) -> str:
|
|
37
|
+
return f'{TAG} type="{label}">\n{BRIEFS[label]}\n</tokencut_task_brief>'
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _typed_messages(segments: list[Segment]) -> dict[int, list[Segment]]:
|
|
41
|
+
"""User text segments grouped by message, in request order (tool results are not text)."""
|
|
42
|
+
out: dict[int, list[Segment]] = {}
|
|
43
|
+
for seg in segments:
|
|
44
|
+
if seg.kind == SegmentKind.USER_TEXT and seg.message_index >= 0:
|
|
45
|
+
out.setdefault(seg.message_index, []).append(seg)
|
|
46
|
+
return dict(sorted(out.items()))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _insert(provider: str, body: dict, path: list, text: str) -> None:
|
|
50
|
+
"""Append the brief to the message at path[:2] (a new block / part, or after a string)."""
|
|
51
|
+
message = body[path[0]][path[1]]
|
|
52
|
+
if provider == "gemini":
|
|
53
|
+
parts = message["parts"]
|
|
54
|
+
last = max(k for k, p in enumerate(parts) if isinstance(p, dict) and "text" in p)
|
|
55
|
+
parts.insert(last + 1, {"text": text})
|
|
56
|
+
return
|
|
57
|
+
content = message["content"]
|
|
58
|
+
if isinstance(content, str):
|
|
59
|
+
message["content"] = f"{content}\n\n{text}"
|
|
60
|
+
return
|
|
61
|
+
last = max(k for k, b in enumerate(content) if isinstance(b, dict) and b.get("type") == "text")
|
|
62
|
+
content.insert(last + 1, {"type": "text", "text": text})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def add_task_briefs(provider: str, body: dict, segments: list[Segment], router,
|
|
66
|
+
threshold: float) -> tuple[list[dict], dict]:
|
|
67
|
+
"""Add briefs to `body` in place. Returns (report entries, task of the newest typed message).
|
|
68
|
+
Side requests (titles, summaries: fewer than two client tools) get nothing. All messages are
|
|
69
|
+
classified before the body is changed, so a router error leaves it untouched."""
|
|
70
|
+
entries: list[dict] = []
|
|
71
|
+
task: dict = {}
|
|
72
|
+
if not _is_agent_request(body):
|
|
73
|
+
return entries, task
|
|
74
|
+
inserts: list[tuple[list, str]] = []
|
|
75
|
+
last_brief: str | None = None
|
|
76
|
+
for index, group in _typed_messages(segments).items():
|
|
77
|
+
joined = "\n".join(seg.text for seg in group)
|
|
78
|
+
if TAG in joined: # already briefed (never twice)
|
|
79
|
+
continue
|
|
80
|
+
text, reason = prepare(joined)
|
|
81
|
+
label, prob = (router.classify(text) if text is not None else (None, None))
|
|
82
|
+
if text is None:
|
|
83
|
+
pass
|
|
84
|
+
elif label == NO_BRIEF:
|
|
85
|
+
reason = "no_brief"
|
|
86
|
+
elif prob < threshold:
|
|
87
|
+
reason = "below_threshold"
|
|
88
|
+
elif label == last_brief:
|
|
89
|
+
reason = "repeat"
|
|
90
|
+
else:
|
|
91
|
+
reason = "brief"
|
|
92
|
+
brief = brief_text(label)
|
|
93
|
+
inserts.append((group[-1].path, brief))
|
|
94
|
+
entries.append({"segment": f"m{index}", "how": "task_brief", "tool": label,
|
|
95
|
+
"before": 0, "after": count_tokens(brief)})
|
|
96
|
+
last_brief = label
|
|
97
|
+
task = {"label": label, "prob": round(prob, 3) if prob is not None else None,
|
|
98
|
+
"briefed": reason == "brief", "reason": reason}
|
|
99
|
+
for path, brief in inserts:
|
|
100
|
+
_insert(provider, body, path, brief)
|
|
101
|
+
return entries, task
|
tokencut/gateway/ccr.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Reversible trimming (CCR): every shortened tool output can be fetched back exactly.
|
|
2
|
+
|
|
3
|
+
A trimmed or masked output gets one extra line naming a reference, e.g. `tc_1a2b3c4d5e6f`.
|
|
4
|
+
The reference is a hash of the ORIGINAL text, so the same output gets the same reference on
|
|
5
|
+
every turn and after a gateway restart (the cached prefix stays byte-for-byte the same).
|
|
6
|
+
|
|
7
|
+
The original is almost always still in the request: agents re-send the full history every
|
|
8
|
+
turn and only TokenCut's outbound copy is shortened. The store below is a fallback for
|
|
9
|
+
originals that left the request (the agent compacted its own history).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import threading
|
|
14
|
+
from collections import OrderedDict
|
|
15
|
+
|
|
16
|
+
from tokencut.gateway.codeview import parse_listing
|
|
17
|
+
|
|
18
|
+
REF_PREFIX = "tc_"
|
|
19
|
+
RETRIEVE_TOOL = "tokencut_retrieve"
|
|
20
|
+
MAX_STORE_BYTES = 64_000_000 # originals kept in RAM (least recently used dropped first)
|
|
21
|
+
MAX_RETRIEVE_LINES = 2_000 # one retrieval returns at most this many lines
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def ref_for(text: str) -> str:
|
|
25
|
+
return REF_PREFIX + hashlib.sha1(text.encode("utf-8", errors="replace")).hexdigest()[:12]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def with_ref(shortened: str, original: str) -> str:
|
|
29
|
+
"""The shortened text plus the line telling the model how to get the original back."""
|
|
30
|
+
ref = ref_for(original)
|
|
31
|
+
return (
|
|
32
|
+
f"{shortened}\n[TokenCut ref {ref}: this output was shortened; "
|
|
33
|
+
f'{RETRIEVE_TOOL}(ref="{ref}", start_line, end_line) returns omitted lines exactly]'
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def slice_lines(original: str, start: int | None, end: int | None) -> tuple[str, int, int]:
|
|
38
|
+
"""Lines start..end (inclusive) of the original, numbered as the model saw them: by the
|
|
39
|
+
tool's own line numbers for a numbered file listing, else by position. Returns
|
|
40
|
+
(text, first, last) with the range actually returned."""
|
|
41
|
+
lines = original.split("\n")
|
|
42
|
+
view = parse_listing(original)
|
|
43
|
+
numbered = any(view.prefixes)
|
|
44
|
+
first_number = view.first_line if numbered else 1
|
|
45
|
+
last_number = first_number + len(lines) - 1
|
|
46
|
+
lo = max(first_number, start or first_number)
|
|
47
|
+
hi = min(last_number, end or last_number, lo + MAX_RETRIEVE_LINES - 1)
|
|
48
|
+
if lo > hi:
|
|
49
|
+
return "", lo, hi
|
|
50
|
+
return "\n".join(lines[lo - first_number: hi - first_number + 1]), lo, hi
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class CCRStore:
|
|
54
|
+
"""Originals by reference, bounded by total size."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, max_bytes: int = MAX_STORE_BYTES):
|
|
57
|
+
self._items: OrderedDict[str, str] = OrderedDict()
|
|
58
|
+
self._bytes = 0
|
|
59
|
+
self._max = max_bytes
|
|
60
|
+
self._lock = threading.Lock() # trimming runs in worker threads
|
|
61
|
+
|
|
62
|
+
def put(self, original: str) -> str:
|
|
63
|
+
ref = ref_for(original)
|
|
64
|
+
with self._lock:
|
|
65
|
+
if ref in self._items:
|
|
66
|
+
self._items.move_to_end(ref)
|
|
67
|
+
return ref
|
|
68
|
+
self._items[ref] = original
|
|
69
|
+
self._bytes += len(original)
|
|
70
|
+
while self._bytes > self._max and len(self._items) > 1:
|
|
71
|
+
_, dropped = self._items.popitem(last=False)
|
|
72
|
+
self._bytes -= len(dropped)
|
|
73
|
+
return ref
|
|
74
|
+
|
|
75
|
+
def get(self, ref: str) -> str | None:
|
|
76
|
+
with self._lock:
|
|
77
|
+
text = self._items.get(ref)
|
|
78
|
+
if text is not None:
|
|
79
|
+
self._items.move_to_end(ref)
|
|
80
|
+
return text
|
|
81
|
+
|
|
82
|
+
def __len__(self) -> int:
|
|
83
|
+
return len(self._items)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Guess what a tool result contains, from the tool's name and the text itself."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from tokencut.contracts import ToolResultKind
|
|
6
|
+
|
|
7
|
+
# Checked in this order; first match wins. Matched against the whole words of the tool name
|
|
8
|
+
# ("read_many_files" -> read, many, files), so "notification" never matches "cat".
|
|
9
|
+
# Names seen in Claude Code, Gemini CLI, Cline, Codex...
|
|
10
|
+
_NAME_RULES: list[tuple[ToolResultKind, frozenset[str]]] = [
|
|
11
|
+
(ToolResultKind.SEARCH, frozenset({"grep", "glob", "search", "find", "ls", "list"})),
|
|
12
|
+
(ToolResultKind.FILE_READ, frozenset({"read", "view", "open", "cat"})),
|
|
13
|
+
(
|
|
14
|
+
ToolResultKind.COMMAND_OUTPUT,
|
|
15
|
+
frozenset({"bash", "shell", "exec", "execute", "run", "terminal", "command", "cmd"}),
|
|
16
|
+
),
|
|
17
|
+
]
|
|
18
|
+
_NAME_WORD = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|[0-9]+")
|
|
19
|
+
# " 12→code", "12\tcode", " 12 | code", "12: code" (line-numbered file listings)
|
|
20
|
+
_NUMBERED_LINE = re.compile(r"^\s*\d+\s*(?:→|\t|\||│|:)")
|
|
21
|
+
_SAMPLE_LINES = 20
|
|
22
|
+
_NUMBERED_SHARE = 0.6
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def looks_like_numbered_file(text: str) -> bool:
|
|
26
|
+
lines = [line for line in text.splitlines() if line.strip()][:_SAMPLE_LINES]
|
|
27
|
+
if len(lines) < 2:
|
|
28
|
+
return False
|
|
29
|
+
numbered = sum(1 for line in lines if _NUMBERED_LINE.match(line))
|
|
30
|
+
return numbered / len(lines) >= _NUMBERED_SHARE
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def classify_tool_result(tool_name: str, text: str) -> ToolResultKind:
|
|
34
|
+
words = {w.lower() for w in _NAME_WORD.findall(tool_name)}
|
|
35
|
+
for kind, rule_words in _NAME_RULES:
|
|
36
|
+
if words & rule_words:
|
|
37
|
+
return kind
|
|
38
|
+
# Unknown tool (e.g. an editor tool's "view" command): fall back to the content.
|
|
39
|
+
if looks_like_numbered_file(text):
|
|
40
|
+
return ToolResultKind.FILE_READ
|
|
41
|
+
return ToolResultKind.OTHER
|