tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Dual-plan demo routing: one Claude Code session, two subscriptions.
|
|
2
|
+
|
|
3
|
+
Claude Code sends every request in Anthropic format. Requests for a Claude model go to
|
|
4
|
+
Anthropic as usual (with the user's Claude plan login). Requests for a GPT model
|
|
5
|
+
(e.g. after `/model gpt-6.1-sol`) go to a local LiteLLM proxy instead, which translates
|
|
6
|
+
them to OpenAI's Responses format and calls the ChatGPT subscription backend with its own
|
|
7
|
+
ChatGPT login. Trimming happens before routing, so both routes are trimmed the same way.
|
|
8
|
+
|
|
9
|
+
Off unless TOKENCUT_GPT_UPSTREAM is set. Demo feature (model routing is ODE's job later).
|
|
10
|
+
|
|
11
|
+
With TOKENCUT_CLAUDE_VIA_LITELLM=1, Claude requests go through the same LiteLLM proxy
|
|
12
|
+
("TokenCut decides, LiteLLM delivers"): the Claude plan login stays in Authorization (LiteLLM
|
|
13
|
+
forwards it to Anthropic) and LiteLLM's own key travels in x-litellm-api-key. If LiteLLM is
|
|
14
|
+
unreachable the gateway sends the Claude request straight to Anthropic (fail open).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
|
|
21
|
+
DEFAULT_GPT_PREFIXES = ("gpt-", "o3", "o4", "codex")
|
|
22
|
+
# The Claude plan login must never be sent to the GPT side.
|
|
23
|
+
_CLAUDE_AUTH_HEADERS = {"authorization", "x-api-key"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class RouteConfig:
|
|
28
|
+
gpt_upstream: str | None = None # e.g. http://127.0.0.1:4000 (LiteLLM proxy)
|
|
29
|
+
gpt_prefixes: tuple[str, ...] = DEFAULT_GPT_PREFIXES
|
|
30
|
+
gpt_key: str | None = None # LiteLLM master key, if the proxy uses one
|
|
31
|
+
claude_via_litellm: bool = False # send Claude requests through LiteLLM as well
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_env(cls) -> "RouteConfig":
|
|
35
|
+
upstream = os.environ.get("TOKENCUT_GPT_UPSTREAM", "").strip().rstrip("/")
|
|
36
|
+
prefixes = os.environ.get("TOKENCUT_GPT_MODELS", "")
|
|
37
|
+
return cls(
|
|
38
|
+
gpt_upstream=upstream or None,
|
|
39
|
+
gpt_prefixes=tuple(p.strip().lower() for p in prefixes.split(",") if p.strip())
|
|
40
|
+
or DEFAULT_GPT_PREFIXES,
|
|
41
|
+
gpt_key=os.environ.get("TOKENCUT_GPT_KEY") or None,
|
|
42
|
+
claude_via_litellm=os.environ.get("TOKENCUT_CLAUDE_VIA_LITELLM", "") == "1",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def model_from_body(body: bytes) -> str:
|
|
47
|
+
"""The request's model name, or "" for anything that is not a JSON object."""
|
|
48
|
+
try:
|
|
49
|
+
data = json.loads(body)
|
|
50
|
+
except (ValueError, UnicodeDecodeError):
|
|
51
|
+
return ""
|
|
52
|
+
return str(data.get("model", "")) if isinstance(data, dict) else ""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def pick_route(provider: str, model: str, config: RouteConfig) -> str:
|
|
56
|
+
""""gpt" or "claude" for Claude Code traffic; other providers keep their own name."""
|
|
57
|
+
if provider != "anthropic":
|
|
58
|
+
return provider
|
|
59
|
+
if config.gpt_upstream and model.lower().startswith(config.gpt_prefixes):
|
|
60
|
+
return "gpt"
|
|
61
|
+
return "claude"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def gpt_headers(headers: dict[str, str], config: RouteConfig) -> dict[str, str]:
|
|
65
|
+
"""Headers for the LiteLLM proxy: without the Claude login, with the proxy key if any."""
|
|
66
|
+
out = {k: v for k, v in headers.items() if k.lower() not in _CLAUDE_AUTH_HEADERS}
|
|
67
|
+
if config.gpt_key:
|
|
68
|
+
out["authorization"] = f"Bearer {config.gpt_key}"
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def claude_via_litellm(config: RouteConfig) -> bool:
|
|
73
|
+
"""Whether Claude inference requests are delivered by LiteLLM (needs its address)."""
|
|
74
|
+
return bool(config.claude_via_litellm and config.gpt_upstream)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def claude_litellm_headers(headers: dict[str, str], config: RouteConfig) -> dict[str, str]:
|
|
78
|
+
"""Headers for a Claude request through LiteLLM: keep the Claude login, add the proxy key."""
|
|
79
|
+
out = dict(headers)
|
|
80
|
+
if config.gpt_key:
|
|
81
|
+
out["x-litellm-api-key"] = f"Bearer {config.gpt_key}"
|
|
82
|
+
return out
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _moved_note(block: dict) -> dict:
|
|
86
|
+
name = str(block.get("title") or "the PDF")
|
|
87
|
+
return {"type": "text", "text": (
|
|
88
|
+
f"[TokenCut: {name} from this tool result follows after the tool results of this message "
|
|
89
|
+
f"(the GPT route does not accept files inside tool results)]")}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def move_tool_documents(body: dict) -> tuple[dict, int]:
|
|
93
|
+
"""(body to send to the GPT route, number of documents moved).
|
|
94
|
+
|
|
95
|
+
ChatGPT's backend rejects a file inside a tool output (400: "expected an image MIME type",
|
|
96
|
+
what LiteLLM makes of an Anthropic `document` in a `tool_result`), but reads the same PDF in
|
|
97
|
+
the user's message. So each document inside a tool result moves to the end of that same
|
|
98
|
+
user message (after its tool results) and leaves a short note in its place. LiteLLM already
|
|
99
|
+
does this for images. Same input -> same output, so it is identical on every turn (prompt
|
|
100
|
+
cache). The input body is not changed."""
|
|
101
|
+
messages = body.get("messages")
|
|
102
|
+
if not isinstance(messages, list):
|
|
103
|
+
return body, 0
|
|
104
|
+
new_messages, moved = [], 0
|
|
105
|
+
for message in messages:
|
|
106
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
107
|
+
if not (isinstance(content, list) and message.get("role") == "user"):
|
|
108
|
+
new_messages.append(message)
|
|
109
|
+
continue
|
|
110
|
+
new_content, documents = [], []
|
|
111
|
+
for block in content:
|
|
112
|
+
inner = block.get("content") if isinstance(block, dict) else None
|
|
113
|
+
if (isinstance(block, dict) and block.get("type") == "tool_result"
|
|
114
|
+
and isinstance(inner, list)
|
|
115
|
+
and any(isinstance(b, dict) and b.get("type") == "document" for b in inner)):
|
|
116
|
+
kept = []
|
|
117
|
+
for b in inner:
|
|
118
|
+
if isinstance(b, dict) and b.get("type") == "document":
|
|
119
|
+
documents.append(b)
|
|
120
|
+
kept.append(_moved_note(b))
|
|
121
|
+
else:
|
|
122
|
+
kept.append(b)
|
|
123
|
+
block = {**block, "content": kept}
|
|
124
|
+
new_content.append(block)
|
|
125
|
+
if documents:
|
|
126
|
+
moved += len(documents)
|
|
127
|
+
message = {**message, "content": new_content + documents}
|
|
128
|
+
new_messages.append(message)
|
|
129
|
+
if not moved:
|
|
130
|
+
return body, 0
|
|
131
|
+
return {**body, "messages": new_messages}, moved
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Per-conversation memory for cache-event trimming.
|
|
2
|
+
|
|
3
|
+
The provider caches the prompt prefix. While it is valid, earlier messages must be sent
|
|
4
|
+
byte-for-byte as before. It is lost anyway when the model changes, after the cache's
|
|
5
|
+
time-to-live, or when the conversation nears the context limit. At those "cache events"
|
|
6
|
+
TokenCut may re-trim the whole history; this module remembers what was decided so every
|
|
7
|
+
later turn sends exactly the same trimmed history until the next event.
|
|
8
|
+
|
|
9
|
+
State lives in memory (local gateway). After a restart the gateway forgets its decisions,
|
|
10
|
+
which costs one cache rewrite — never a broken request.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import threading
|
|
15
|
+
from collections import OrderedDict
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
19
|
+
|
|
20
|
+
MAX_CONVERSATIONS = 500 # least recently used conversations are forgotten first
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def fingerprint(seg: Segment) -> str:
|
|
24
|
+
"""Identity of a segment's ORIGINAL content (stable across turns)."""
|
|
25
|
+
raw = f"{seg.kind.value}\0{seg.tool_call_id}\0{seg.text}"
|
|
26
|
+
return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def conversation_key(provider: str, segments: list[Segment]) -> str | None:
|
|
30
|
+
"""Same for every turn of one conversation, whatever model is used: system prompt +
|
|
31
|
+
first user message. None if the request has no user message. System context the agent
|
|
32
|
+
adds later in the conversation (Claude Code's role "system" messages) is left out."""
|
|
33
|
+
first = next((k for k, s in enumerate(segments) if s.kind == SegmentKind.USER_TEXT), None)
|
|
34
|
+
if first is None:
|
|
35
|
+
return None
|
|
36
|
+
system = [s.text for s in segments[:first] if s.kind == SegmentKind.SYSTEM]
|
|
37
|
+
first_user = segments[first].text
|
|
38
|
+
raw = "\0".join([provider, *system, first_user[:4000]])
|
|
39
|
+
return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class ConversationState:
|
|
44
|
+
model: str
|
|
45
|
+
last_seen: float
|
|
46
|
+
pinned: dict[str, str] = field(default_factory=dict) # fingerprint -> text sent instead
|
|
47
|
+
masked_paths: set[str] = field(default_factory=set) # files whose old reads were removed
|
|
48
|
+
protected_paths: set[str] = field(default_factory=set) # re-read after removal: keep
|
|
49
|
+
kept_tools: list[str] | None = None # tool names offered since the last event
|
|
50
|
+
model_seen: dict[str, float] = field(default_factory=dict) # model -> last request time
|
|
51
|
+
last_event_tokens: int | None = None # request size at the last event
|
|
52
|
+
events: int = 0
|
|
53
|
+
user_turns: int = 0 # real user messages in the last request (a new one = a new task)
|
|
54
|
+
requests: int = 0 # requests seen in this conversation
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class SessionStore:
|
|
58
|
+
def __init__(self, max_conversations: int = MAX_CONVERSATIONS):
|
|
59
|
+
self._states: OrderedDict[str, ConversationState] = OrderedDict()
|
|
60
|
+
self._max = max_conversations
|
|
61
|
+
self.lock = threading.Lock() # trimming runs in worker threads
|
|
62
|
+
|
|
63
|
+
def get(self, key: str) -> ConversationState | None:
|
|
64
|
+
state = self._states.get(key)
|
|
65
|
+
if state is not None:
|
|
66
|
+
self._states.move_to_end(key)
|
|
67
|
+
return state
|
|
68
|
+
|
|
69
|
+
def put(self, key: str, state: ConversationState) -> None:
|
|
70
|
+
self._states[key] = state
|
|
71
|
+
self._states.move_to_end(key)
|
|
72
|
+
while len(self._states) > self._max:
|
|
73
|
+
self._states.popitem(last=False)
|
|
74
|
+
|
|
75
|
+
def __len__(self) -> int:
|
|
76
|
+
return len(self._states)
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Task type of a typed user message (prompt enhancement): rules first, then the trained router.
|
|
2
|
+
|
|
3
|
+
Rules: `<system-reminder>` text is removed; machine-generated messages (slash commands,
|
|
4
|
+
interrupts, harness / CI / sub-agent / approval messages, tool logs, image-only) and short
|
|
5
|
+
follow-ups ("yes", "continue") get no brief. Everything else goes to the router: bge-small
|
|
6
|
+
fine-tuned on SWE-chat (RESULTS_REPORT §11), run as ONNX on CPU with the training input (first
|
|
7
|
+
360 + "…" + last 149 tokens) and probabilities re-weighted to the natural label mix.
|
|
8
|
+
|
|
9
|
+
The model files (router.onnx, router.onnx.data, tokenizer.json, router_config.json) are not in
|
|
10
|
+
git: TOKENCUT_ROUTER_DIR, or downloaded once (no login) from the public Hugging Face repo
|
|
11
|
+
TOKENCUT_ROUTER_REPO. onnxruntime, tokenizers and huggingface_hub are package dependencies and
|
|
12
|
+
are imported only when the router is used.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import re
|
|
19
|
+
from collections import OrderedDict
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
from tokencut.gateway.prompt_router import rule_label
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# Must stay identical to eval/prompt_router/router_metrics.py (the standalone Kaggle copy).
|
|
29
|
+
LABELS = ["bugfix", "explain", "feature", "no_brief", "refactor", "run_action", "tests"]
|
|
30
|
+
NO_BRIEF = "no_brief"
|
|
31
|
+
HEAD_TOKENS, TAIL_TOKENS = 360, 149
|
|
32
|
+
MODEL_FILES = ("router.onnx", "router.onnx.data", "tokenizer.json", "router_config.json")
|
|
33
|
+
DEFAULT_REPO = "humataj/tokencut-router"
|
|
34
|
+
DEFAULT_CACHE = Path.home() / ".cache" / "tokencut" / "router"
|
|
35
|
+
CACHE_SIZE = 10_000 # classified messages kept (text hash -> label, prob): ~1 MB
|
|
36
|
+
|
|
37
|
+
_REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
|
|
38
|
+
_MACHINE_START = re.compile(
|
|
39
|
+
r"^[\s\"'`]*(<(command-name|command-message|command-args|local-command-stdout|environment_context|"
|
|
40
|
+
r"subagent_notification|turn_aborted|skill|skills|codex_internal_context|recommended_plugins|"
|
|
41
|
+
r"timestamp|task-notification|teammate-message|ci-monitor-event|objective|state_management|"
|
|
42
|
+
r"verification|team_[a-z_]+|system-reminder|user_instructions|permissions)\b"
|
|
43
|
+
r"|\[\d+\] tool [\w.-]+ (call|result):|\[\d+\] (assistant|user|system):"
|
|
44
|
+
r"|Base directory for this skill:|Current state: \w+ \(for " # Claude Code status summary
|
|
45
|
+
r"|\[Request interrupted|\[delegate worktree\]|/[A-Za-z][\w:-]*(\s|$)|>>> )",
|
|
46
|
+
re.IGNORECASE)
|
|
47
|
+
_MACHINE_ANYWHERE = re.compile(
|
|
48
|
+
r"APPROVAL REQUEST START|The Codex agent has requested|TRANSCRIPT DELTA START|"
|
|
49
|
+
r"^Execution instructions:|^# AGENTS\.md instructions|Another Claude session sent a message|"
|
|
50
|
+
r"<teammate-message|<task-notification|<usage>|<tool_uses>", re.IGNORECASE | re.MULTILINE)
|
|
51
|
+
_IMAGE = re.compile(r"<image[^>]*>|</image>|\[Image[^\]]*\]", re.IGNORECASE)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def strip_markup(text: str) -> str:
|
|
55
|
+
return " ".join(_IMAGE.sub(" ", text).split()) if _IMAGE.search(text) else text.strip()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def is_machine(text: str) -> bool:
|
|
59
|
+
"""Not typed by the user: harness, slash command, CI / sub-agent notice, approval request."""
|
|
60
|
+
rest = strip_markup(text)
|
|
61
|
+
return (not rest or bool(_MACHINE_START.match(text)) or bool(_MACHINE_ANYWHERE.search(text)))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prepare(text: str) -> tuple[str | None, str]:
|
|
65
|
+
"""(text for the router, "") or (None, reason the message gets no brief)."""
|
|
66
|
+
text = _REMINDER.sub("", text).strip()
|
|
67
|
+
if not text:
|
|
68
|
+
return None, "empty"
|
|
69
|
+
if is_machine(text):
|
|
70
|
+
return None, "machine"
|
|
71
|
+
if rule_label(text) == "followup":
|
|
72
|
+
return None, "followup"
|
|
73
|
+
return strip_markup(text), ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def cut_token_ids(ids: list[int], head: int = HEAD_TOKENS, tail: int = TAIL_TOKENS,
|
|
77
|
+
marker: list[int] | None = None) -> list[int]:
|
|
78
|
+
"""Keep the first `head` and last `tail` token ids (instruction + final error line)."""
|
|
79
|
+
if len(ids) <= head + tail + len(marker or []):
|
|
80
|
+
return list(ids)
|
|
81
|
+
return list(ids[:head]) + list(marker or []) + list(ids[-tail:])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def prior_correct(probs: np.ndarray, train_prior: np.ndarray, real_prior: np.ndarray) -> np.ndarray:
|
|
85
|
+
"""Re-weight probabilities from the training label mix to the real one (Bayes' rule)."""
|
|
86
|
+
scaled = probs * (np.asarray(real_prior) / np.maximum(np.asarray(train_prior), 1e-12))
|
|
87
|
+
return scaled / scaled.sum(axis=1, keepdims=True)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class OnnxRouter:
|
|
91
|
+
"""Exactly the training input: head 6,000 + tail 2,000 characters for huge pastes, then the
|
|
92
|
+
first 360 + "…" + last 149 tokens between [CLS] and [SEP]."""
|
|
93
|
+
|
|
94
|
+
def __init__(self, folder: Path):
|
|
95
|
+
import onnxruntime as ort
|
|
96
|
+
from tokenizers import Tokenizer
|
|
97
|
+
|
|
98
|
+
self.config = json.loads((folder / "router_config.json").read_text(encoding="utf-8"))
|
|
99
|
+
if self.config["labels"] != LABELS:
|
|
100
|
+
raise ValueError(f"router labels {self.config['labels']} != {LABELS}")
|
|
101
|
+
self.tokenizer = Tokenizer.from_file(str(folder / "tokenizer.json"))
|
|
102
|
+
self.tokenizer.no_truncation()
|
|
103
|
+
self.tokenizer.no_padding()
|
|
104
|
+
self.marker = [self.tokenizer.token_to_id("…")]
|
|
105
|
+
self.cls, self.sep = self.tokenizer.token_to_id("[CLS]"), self.tokenizer.token_to_id("[SEP]")
|
|
106
|
+
options = ort.SessionOptions()
|
|
107
|
+
options.intra_op_num_threads = 4
|
|
108
|
+
self.session = ort.InferenceSession(str(folder / "router.onnx"), options,
|
|
109
|
+
providers=["CPUExecutionProvider"])
|
|
110
|
+
self.train_prior = np.array(self.config["train_prior"])
|
|
111
|
+
self.natural_prior = np.array(self.config["natural_prior"])
|
|
112
|
+
self._cache: OrderedDict[str, tuple[str, float]] = OrderedDict()
|
|
113
|
+
|
|
114
|
+
def ids(self, text: str) -> list[int]:
|
|
115
|
+
text = text[:6000] + ("\n" + text[-2000:] if len(text) > 8000 else "")
|
|
116
|
+
ids = self.tokenizer.encode(text, add_special_tokens=False).ids
|
|
117
|
+
cut = cut_token_ids(ids, self.config["head_tokens"], self.config["tail_tokens"], self.marker)
|
|
118
|
+
return [self.cls, *cut, self.sep]
|
|
119
|
+
|
|
120
|
+
def probs(self, text: str) -> np.ndarray:
|
|
121
|
+
"""Raw model probabilities (no prior correction)."""
|
|
122
|
+
ids = np.array([self.ids(text)], dtype=np.int64)
|
|
123
|
+
logits = self.session.run(None, {"input_ids": ids, "attention_mask": np.ones_like(ids)})[0]
|
|
124
|
+
e = np.exp(logits - logits.max(axis=1, keepdims=True))
|
|
125
|
+
return (e / e.sum(axis=1, keepdims=True))[0]
|
|
126
|
+
|
|
127
|
+
def classify(self, text: str) -> tuple[str, float]:
|
|
128
|
+
"""(label, probability) for a prepared text, prior-corrected; cached by text hash."""
|
|
129
|
+
key = hashlib.sha1(text.encode("utf-8")).hexdigest()
|
|
130
|
+
if key in self._cache:
|
|
131
|
+
self._cache.move_to_end(key)
|
|
132
|
+
return self._cache[key]
|
|
133
|
+
probs = prior_correct(self.probs(text)[None, :], self.train_prior, self.natural_prior)[0]
|
|
134
|
+
best = int(probs.argmax())
|
|
135
|
+
result = (LABELS[best], float(probs[best]))
|
|
136
|
+
self._cache[key] = result
|
|
137
|
+
if len(self._cache) > CACHE_SIZE:
|
|
138
|
+
self._cache.popitem(last=False)
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def load_router(folder: str | None = None, repo: str | None = None) -> OnnxRouter | None:
|
|
143
|
+
"""The router from a local folder, else from the Hugging Face repo; None if unavailable
|
|
144
|
+
(the gateway then adds no briefs: fail open)."""
|
|
145
|
+
try:
|
|
146
|
+
path = Path(folder).expanduser() if folder else None
|
|
147
|
+
if path is None or not all((path / f).exists() for f in MODEL_FILES):
|
|
148
|
+
from huggingface_hub import snapshot_download
|
|
149
|
+
|
|
150
|
+
path = Path(snapshot_download(repo_id=repo or DEFAULT_REPO, allow_patterns=list(MODEL_FILES),
|
|
151
|
+
local_dir=str(DEFAULT_CACHE)))
|
|
152
|
+
router = OnnxRouter(path)
|
|
153
|
+
log.info("prompt router loaded from %s", path)
|
|
154
|
+
return router
|
|
155
|
+
except Exception:
|
|
156
|
+
log.warning("prompt router unavailable; no task briefs", exc_info=True)
|
|
157
|
+
return None
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Trim long command output (test runs, builds) and search results.
|
|
2
|
+
|
|
3
|
+
Keeps the lines that matter most for fixing code, in this priority order, until the token
|
|
4
|
+
budget is used: the end of the output (test summaries, final error), lines that look like
|
|
5
|
+
errors with a little context, then the beginning. Every dropped run becomes one marker.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
from tokencut.tokens import count_tokens
|
|
11
|
+
|
|
12
|
+
_ERROR = re.compile(
|
|
13
|
+
r"error|exception|traceback|failed|failure|assert|fatal|panic|warning|"
|
|
14
|
+
r"\bE\s{2,}|^\s*File \"|^FAIL|^ERROR",
|
|
15
|
+
re.IGNORECASE,
|
|
16
|
+
)
|
|
17
|
+
TAIL_LINES = 40
|
|
18
|
+
HEAD_LINES = 15
|
|
19
|
+
ERROR_CONTEXT = 2 # lines kept before and after an error line
|
|
20
|
+
# Tables (CSV / TSV / Markdown) are never cut with head / tail: the answer can sit in any
|
|
21
|
+
# middle row (docs/ATTACHMENTS_PLAN.md, step 2).
|
|
22
|
+
TABLE_SAMPLE_LINES = 40
|
|
23
|
+
TABLE_MIN_LINES = 5
|
|
24
|
+
TABLE_SHARE = 0.8 # share of sampled lines with the same number of delimiters
|
|
25
|
+
_TABLE_DELIMITERS = (",", "\t", "|", ";")
|
|
26
|
+
_READ_PREFIX = re.compile(r"^\s*\d+→") # Claude Code's Read line numbers
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def looks_like_table(text: str) -> bool:
|
|
30
|
+
"""Most lines carry the same number (>= 2) of one delimiter: CSV, TSV, Markdown table."""
|
|
31
|
+
lines = [_READ_PREFIX.sub("", line, count=1) for line in text.split("\n")]
|
|
32
|
+
lines = [line for line in lines if line.strip()][:TABLE_SAMPLE_LINES]
|
|
33
|
+
if len(lines) < TABLE_MIN_LINES:
|
|
34
|
+
return False
|
|
35
|
+
for delimiter in _TABLE_DELIMITERS:
|
|
36
|
+
counts = [line.count(delimiter) for line in lines]
|
|
37
|
+
common = max(sorted(set(counts)), key=counts.count)
|
|
38
|
+
if common >= 2 and counts.count(common) >= TABLE_SHARE * len(lines):
|
|
39
|
+
return True
|
|
40
|
+
return False
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _priority_order(lines: list[str]) -> list[int]:
|
|
44
|
+
"""Line indexes in the order we want to keep them (no duplicates)."""
|
|
45
|
+
order: list[int] = []
|
|
46
|
+
seen: set[int] = set()
|
|
47
|
+
|
|
48
|
+
def push(i: int) -> None:
|
|
49
|
+
if 0 <= i < len(lines) and i not in seen:
|
|
50
|
+
seen.add(i)
|
|
51
|
+
order.append(i)
|
|
52
|
+
|
|
53
|
+
for i in range(len(lines) - 1, max(-1, len(lines) - 1 - TAIL_LINES), -1):
|
|
54
|
+
push(i)
|
|
55
|
+
for i, line in enumerate(lines):
|
|
56
|
+
if _ERROR.search(line):
|
|
57
|
+
for k in range(i - ERROR_CONTEXT, i + ERROR_CONTEXT + 1):
|
|
58
|
+
push(k)
|
|
59
|
+
for i in range(min(HEAD_LINES, len(lines))):
|
|
60
|
+
push(i)
|
|
61
|
+
return order
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def trim_output(text: str, budget_tokens: int, what: str = "output") -> str | None:
|
|
65
|
+
"""Trimmed text within roughly budget_tokens, or None if no trim is needed/possible."""
|
|
66
|
+
if count_tokens(text) <= budget_tokens:
|
|
67
|
+
return None
|
|
68
|
+
lines = text.split("\n")
|
|
69
|
+
keep: set[int] = set()
|
|
70
|
+
used = 0
|
|
71
|
+
for i in _priority_order(lines):
|
|
72
|
+
cost = count_tokens(lines[i]) + 1
|
|
73
|
+
if used + cost > budget_tokens:
|
|
74
|
+
continue # a shorter line further down the priority list may still fit
|
|
75
|
+
keep.add(i)
|
|
76
|
+
used += cost
|
|
77
|
+
|
|
78
|
+
out: list[str] = []
|
|
79
|
+
i = 0
|
|
80
|
+
while i < len(lines):
|
|
81
|
+
if i in keep:
|
|
82
|
+
out.append(lines[i])
|
|
83
|
+
i += 1
|
|
84
|
+
continue
|
|
85
|
+
j = i
|
|
86
|
+
while j < len(lines) and j not in keep:
|
|
87
|
+
j += 1
|
|
88
|
+
out.append(f"[TokenCut: {j - i} lines of {what} omitted]")
|
|
89
|
+
i = j
|
|
90
|
+
result = "\n".join(out)
|
|
91
|
+
return result if count_tokens(result) < count_tokens(text) else None
|