tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,131 @@
1
+ """Dual-plan demo routing: one Claude Code session, two subscriptions.
2
+
3
+ Claude Code sends every request in Anthropic format. Requests for a Claude model go to
4
+ Anthropic as usual (with the user's Claude plan login). Requests for a GPT model
5
+ (e.g. after `/model gpt-6.1-sol`) go to a local LiteLLM proxy instead, which translates
6
+ them to OpenAI's Responses format and calls the ChatGPT subscription backend with its own
7
+ ChatGPT login. Trimming happens before routing, so both routes are trimmed the same way.
8
+
9
+ Off unless TOKENCUT_GPT_UPSTREAM is set. Demo feature (model routing is ODE's job later).
10
+
11
+ With TOKENCUT_CLAUDE_VIA_LITELLM=1, Claude requests go through the same LiteLLM proxy
12
+ ("TokenCut decides, LiteLLM delivers"): the Claude plan login stays in Authorization (LiteLLM
13
+ forwards it to Anthropic) and LiteLLM's own key travels in x-litellm-api-key. If LiteLLM is
14
+ unreachable the gateway sends the Claude request straight to Anthropic (fail open).
15
+ """
16
+
17
+ import json
18
+ import os
19
+ from dataclasses import dataclass
20
+
21
+ DEFAULT_GPT_PREFIXES = ("gpt-", "o3", "o4", "codex")
22
+ # The Claude plan login must never be sent to the GPT side.
23
+ _CLAUDE_AUTH_HEADERS = {"authorization", "x-api-key"}
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class RouteConfig:
28
+ gpt_upstream: str | None = None # e.g. http://127.0.0.1:4000 (LiteLLM proxy)
29
+ gpt_prefixes: tuple[str, ...] = DEFAULT_GPT_PREFIXES
30
+ gpt_key: str | None = None # LiteLLM master key, if the proxy uses one
31
+ claude_via_litellm: bool = False # send Claude requests through LiteLLM as well
32
+
33
+ @classmethod
34
+ def from_env(cls) -> "RouteConfig":
35
+ upstream = os.environ.get("TOKENCUT_GPT_UPSTREAM", "").strip().rstrip("/")
36
+ prefixes = os.environ.get("TOKENCUT_GPT_MODELS", "")
37
+ return cls(
38
+ gpt_upstream=upstream or None,
39
+ gpt_prefixes=tuple(p.strip().lower() for p in prefixes.split(",") if p.strip())
40
+ or DEFAULT_GPT_PREFIXES,
41
+ gpt_key=os.environ.get("TOKENCUT_GPT_KEY") or None,
42
+ claude_via_litellm=os.environ.get("TOKENCUT_CLAUDE_VIA_LITELLM", "") == "1",
43
+ )
44
+
45
+
46
+ def model_from_body(body: bytes) -> str:
47
+ """The request's model name, or "" for anything that is not a JSON object."""
48
+ try:
49
+ data = json.loads(body)
50
+ except (ValueError, UnicodeDecodeError):
51
+ return ""
52
+ return str(data.get("model", "")) if isinstance(data, dict) else ""
53
+
54
+
55
+ def pick_route(provider: str, model: str, config: RouteConfig) -> str:
56
+ """"gpt" or "claude" for Claude Code traffic; other providers keep their own name."""
57
+ if provider != "anthropic":
58
+ return provider
59
+ if config.gpt_upstream and model.lower().startswith(config.gpt_prefixes):
60
+ return "gpt"
61
+ return "claude"
62
+
63
+
64
+ def gpt_headers(headers: dict[str, str], config: RouteConfig) -> dict[str, str]:
65
+ """Headers for the LiteLLM proxy: without the Claude login, with the proxy key if any."""
66
+ out = {k: v for k, v in headers.items() if k.lower() not in _CLAUDE_AUTH_HEADERS}
67
+ if config.gpt_key:
68
+ out["authorization"] = f"Bearer {config.gpt_key}"
69
+ return out
70
+
71
+
72
+ def claude_via_litellm(config: RouteConfig) -> bool:
73
+ """Whether Claude inference requests are delivered by LiteLLM (needs its address)."""
74
+ return bool(config.claude_via_litellm and config.gpt_upstream)
75
+
76
+
77
+ def claude_litellm_headers(headers: dict[str, str], config: RouteConfig) -> dict[str, str]:
78
+ """Headers for a Claude request through LiteLLM: keep the Claude login, add the proxy key."""
79
+ out = dict(headers)
80
+ if config.gpt_key:
81
+ out["x-litellm-api-key"] = f"Bearer {config.gpt_key}"
82
+ return out
83
+
84
+
85
+ def _moved_note(block: dict) -> dict:
86
+ name = str(block.get("title") or "the PDF")
87
+ return {"type": "text", "text": (
88
+ f"[TokenCut: {name} from this tool result follows after the tool results of this message "
89
+ f"(the GPT route does not accept files inside tool results)]")}
90
+
91
+
92
+ def move_tool_documents(body: dict) -> tuple[dict, int]:
93
+ """(body to send to the GPT route, number of documents moved).
94
+
95
+ ChatGPT's backend rejects a file inside a tool output (400: "expected an image MIME type",
96
+ what LiteLLM makes of an Anthropic `document` in a `tool_result`), but reads the same PDF in
97
+ the user's message. So each document inside a tool result moves to the end of that same
98
+ user message (after its tool results) and leaves a short note in its place. LiteLLM already
99
+ does this for images. Same input -> same output, so it is identical on every turn (prompt
100
+ cache). The input body is not changed."""
101
+ messages = body.get("messages")
102
+ if not isinstance(messages, list):
103
+ return body, 0
104
+ new_messages, moved = [], 0
105
+ for message in messages:
106
+ content = message.get("content") if isinstance(message, dict) else None
107
+ if not (isinstance(content, list) and message.get("role") == "user"):
108
+ new_messages.append(message)
109
+ continue
110
+ new_content, documents = [], []
111
+ for block in content:
112
+ inner = block.get("content") if isinstance(block, dict) else None
113
+ if (isinstance(block, dict) and block.get("type") == "tool_result"
114
+ and isinstance(inner, list)
115
+ and any(isinstance(b, dict) and b.get("type") == "document" for b in inner)):
116
+ kept = []
117
+ for b in inner:
118
+ if isinstance(b, dict) and b.get("type") == "document":
119
+ documents.append(b)
120
+ kept.append(_moved_note(b))
121
+ else:
122
+ kept.append(b)
123
+ block = {**block, "content": kept}
124
+ new_content.append(block)
125
+ if documents:
126
+ moved += len(documents)
127
+ message = {**message, "content": new_content + documents}
128
+ new_messages.append(message)
129
+ if not moved:
130
+ return body, 0
131
+ return {**body, "messages": new_messages}, moved
@@ -0,0 +1,76 @@
1
+ """Per-conversation memory for cache-event trimming.
2
+
3
+ The provider caches the prompt prefix. While it is valid, earlier messages must be sent
4
+ byte-for-byte as before. It is lost anyway when the model changes, after the cache's
5
+ time-to-live, or when the conversation nears the context limit. At those "cache events"
6
+ TokenCut may re-trim the whole history; this module remembers what was decided so every
7
+ later turn sends exactly the same trimmed history until the next event.
8
+
9
+ State lives in memory (local gateway). After a restart the gateway forgets its decisions,
10
+ which costs one cache rewrite — never a broken request.
11
+ """
12
+
13
+ import hashlib
14
+ import threading
15
+ from collections import OrderedDict
16
+ from dataclasses import dataclass, field
17
+
18
+ from tokencut.contracts import Segment, SegmentKind
19
+
20
+ MAX_CONVERSATIONS = 500 # least recently used conversations are forgotten first
21
+
22
+
23
+ def fingerprint(seg: Segment) -> str:
24
+ """Identity of a segment's ORIGINAL content (stable across turns)."""
25
+ raw = f"{seg.kind.value}\0{seg.tool_call_id}\0{seg.text}"
26
+ return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest()
27
+
28
+
29
+ def conversation_key(provider: str, segments: list[Segment]) -> str | None:
30
+ """Same for every turn of one conversation, whatever model is used: system prompt +
31
+ first user message. None if the request has no user message. System context the agent
32
+ adds later in the conversation (Claude Code's role "system" messages) is left out."""
33
+ first = next((k for k, s in enumerate(segments) if s.kind == SegmentKind.USER_TEXT), None)
34
+ if first is None:
35
+ return None
36
+ system = [s.text for s in segments[:first] if s.kind == SegmentKind.SYSTEM]
37
+ first_user = segments[first].text
38
+ raw = "\0".join([provider, *system, first_user[:4000]])
39
+ return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest()
40
+
41
+
42
+ @dataclass
43
+ class ConversationState:
44
+ model: str
45
+ last_seen: float
46
+ pinned: dict[str, str] = field(default_factory=dict) # fingerprint -> text sent instead
47
+ masked_paths: set[str] = field(default_factory=set) # files whose old reads were removed
48
+ protected_paths: set[str] = field(default_factory=set) # re-read after removal: keep
49
+ kept_tools: list[str] | None = None # tool names offered since the last event
50
+ model_seen: dict[str, float] = field(default_factory=dict) # model -> last request time
51
+ last_event_tokens: int | None = None # request size at the last event
52
+ events: int = 0
53
+ user_turns: int = 0 # real user messages in the last request (a new one = a new task)
54
+ requests: int = 0 # requests seen in this conversation
55
+
56
+
57
+ class SessionStore:
58
+ def __init__(self, max_conversations: int = MAX_CONVERSATIONS):
59
+ self._states: OrderedDict[str, ConversationState] = OrderedDict()
60
+ self._max = max_conversations
61
+ self.lock = threading.Lock() # trimming runs in worker threads
62
+
63
+ def get(self, key: str) -> ConversationState | None:
64
+ state = self._states.get(key)
65
+ if state is not None:
66
+ self._states.move_to_end(key)
67
+ return state
68
+
69
+ def put(self, key: str, state: ConversationState) -> None:
70
+ self._states[key] = state
71
+ self._states.move_to_end(key)
72
+ while len(self._states) > self._max:
73
+ self._states.popitem(last=False)
74
+
75
+ def __len__(self) -> int:
76
+ return len(self._states)
@@ -0,0 +1,157 @@
1
+ """Task type of a typed user message (prompt enhancement): rules first, then the trained router.
2
+
3
+ Rules: `<system-reminder>` text is removed; machine-generated messages (slash commands,
4
+ interrupts, harness / CI / sub-agent / approval messages, tool logs, image-only) and short
5
+ follow-ups ("yes", "continue") get no brief. Everything else goes to the router: bge-small
6
+ fine-tuned on SWE-chat (RESULTS_REPORT §11), run as ONNX on CPU with the training input (first
7
+ 360 + "…" + last 149 tokens) and probabilities re-weighted to the natural label mix.
8
+
9
+ The model files (router.onnx, router.onnx.data, tokenizer.json, router_config.json) are not in
10
+ git: TOKENCUT_ROUTER_DIR, or downloaded once (no login) from the public Hugging Face repo
11
+ TOKENCUT_ROUTER_REPO. onnxruntime, tokenizers and huggingface_hub are package dependencies and
12
+ are imported only when the router is used.
13
+ """
14
+
15
+ import hashlib
16
+ import json
17
+ import logging
18
+ import re
19
+ from collections import OrderedDict
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+
24
+ from tokencut.gateway.prompt_router import rule_label
25
+
26
+ log = logging.getLogger(__name__)
27
+
28
+ # Must stay identical to eval/prompt_router/router_metrics.py (the standalone Kaggle copy).
29
+ LABELS = ["bugfix", "explain", "feature", "no_brief", "refactor", "run_action", "tests"]
30
+ NO_BRIEF = "no_brief"
31
+ HEAD_TOKENS, TAIL_TOKENS = 360, 149
32
+ MODEL_FILES = ("router.onnx", "router.onnx.data", "tokenizer.json", "router_config.json")
33
+ DEFAULT_REPO = "humataj/tokencut-router"
34
+ DEFAULT_CACHE = Path.home() / ".cache" / "tokencut" / "router"
35
+ CACHE_SIZE = 10_000 # classified messages kept (text hash -> label, prob): ~1 MB
36
+
37
+ _REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
38
+ _MACHINE_START = re.compile(
39
+ r"^[\s\"'`]*(<(command-name|command-message|command-args|local-command-stdout|environment_context|"
40
+ r"subagent_notification|turn_aborted|skill|skills|codex_internal_context|recommended_plugins|"
41
+ r"timestamp|task-notification|teammate-message|ci-monitor-event|objective|state_management|"
42
+ r"verification|team_[a-z_]+|system-reminder|user_instructions|permissions)\b"
43
+ r"|\[\d+\] tool [\w.-]+ (call|result):|\[\d+\] (assistant|user|system):"
44
+ r"|Base directory for this skill:|Current state: \w+ \(for " # Claude Code status summary
45
+ r"|\[Request interrupted|\[delegate worktree\]|/[A-Za-z][\w:-]*(\s|$)|>>> )",
46
+ re.IGNORECASE)
47
+ _MACHINE_ANYWHERE = re.compile(
48
+ r"APPROVAL REQUEST START|The Codex agent has requested|TRANSCRIPT DELTA START|"
49
+ r"^Execution instructions:|^# AGENTS\.md instructions|Another Claude session sent a message|"
50
+ r"<teammate-message|<task-notification|<usage>|<tool_uses>", re.IGNORECASE | re.MULTILINE)
51
+ _IMAGE = re.compile(r"<image[^>]*>|</image>|\[Image[^\]]*\]", re.IGNORECASE)
52
+
53
+
54
+ def strip_markup(text: str) -> str:
55
+ return " ".join(_IMAGE.sub(" ", text).split()) if _IMAGE.search(text) else text.strip()
56
+
57
+
58
+ def is_machine(text: str) -> bool:
59
+ """Not typed by the user: harness, slash command, CI / sub-agent notice, approval request."""
60
+ rest = strip_markup(text)
61
+ return (not rest or bool(_MACHINE_START.match(text)) or bool(_MACHINE_ANYWHERE.search(text)))
62
+
63
+
64
+ def prepare(text: str) -> tuple[str | None, str]:
65
+ """(text for the router, "") or (None, reason the message gets no brief)."""
66
+ text = _REMINDER.sub("", text).strip()
67
+ if not text:
68
+ return None, "empty"
69
+ if is_machine(text):
70
+ return None, "machine"
71
+ if rule_label(text) == "followup":
72
+ return None, "followup"
73
+ return strip_markup(text), ""
74
+
75
+
76
+ def cut_token_ids(ids: list[int], head: int = HEAD_TOKENS, tail: int = TAIL_TOKENS,
77
+ marker: list[int] | None = None) -> list[int]:
78
+ """Keep the first `head` and last `tail` token ids (instruction + final error line)."""
79
+ if len(ids) <= head + tail + len(marker or []):
80
+ return list(ids)
81
+ return list(ids[:head]) + list(marker or []) + list(ids[-tail:])
82
+
83
+
84
+ def prior_correct(probs: np.ndarray, train_prior: np.ndarray, real_prior: np.ndarray) -> np.ndarray:
85
+ """Re-weight probabilities from the training label mix to the real one (Bayes' rule)."""
86
+ scaled = probs * (np.asarray(real_prior) / np.maximum(np.asarray(train_prior), 1e-12))
87
+ return scaled / scaled.sum(axis=1, keepdims=True)
88
+
89
+
90
+ class OnnxRouter:
91
+ """Exactly the training input: head 6,000 + tail 2,000 characters for huge pastes, then the
92
+ first 360 + "…" + last 149 tokens between [CLS] and [SEP]."""
93
+
94
+ def __init__(self, folder: Path):
95
+ import onnxruntime as ort
96
+ from tokenizers import Tokenizer
97
+
98
+ self.config = json.loads((folder / "router_config.json").read_text(encoding="utf-8"))
99
+ if self.config["labels"] != LABELS:
100
+ raise ValueError(f"router labels {self.config['labels']} != {LABELS}")
101
+ self.tokenizer = Tokenizer.from_file(str(folder / "tokenizer.json"))
102
+ self.tokenizer.no_truncation()
103
+ self.tokenizer.no_padding()
104
+ self.marker = [self.tokenizer.token_to_id("…")]
105
+ self.cls, self.sep = self.tokenizer.token_to_id("[CLS]"), self.tokenizer.token_to_id("[SEP]")
106
+ options = ort.SessionOptions()
107
+ options.intra_op_num_threads = 4
108
+ self.session = ort.InferenceSession(str(folder / "router.onnx"), options,
109
+ providers=["CPUExecutionProvider"])
110
+ self.train_prior = np.array(self.config["train_prior"])
111
+ self.natural_prior = np.array(self.config["natural_prior"])
112
+ self._cache: OrderedDict[str, tuple[str, float]] = OrderedDict()
113
+
114
+ def ids(self, text: str) -> list[int]:
115
+ text = text[:6000] + ("\n" + text[-2000:] if len(text) > 8000 else "")
116
+ ids = self.tokenizer.encode(text, add_special_tokens=False).ids
117
+ cut = cut_token_ids(ids, self.config["head_tokens"], self.config["tail_tokens"], self.marker)
118
+ return [self.cls, *cut, self.sep]
119
+
120
+ def probs(self, text: str) -> np.ndarray:
121
+ """Raw model probabilities (no prior correction)."""
122
+ ids = np.array([self.ids(text)], dtype=np.int64)
123
+ logits = self.session.run(None, {"input_ids": ids, "attention_mask": np.ones_like(ids)})[0]
124
+ e = np.exp(logits - logits.max(axis=1, keepdims=True))
125
+ return (e / e.sum(axis=1, keepdims=True))[0]
126
+
127
+ def classify(self, text: str) -> tuple[str, float]:
128
+ """(label, probability) for a prepared text, prior-corrected; cached by text hash."""
129
+ key = hashlib.sha1(text.encode("utf-8")).hexdigest()
130
+ if key in self._cache:
131
+ self._cache.move_to_end(key)
132
+ return self._cache[key]
133
+ probs = prior_correct(self.probs(text)[None, :], self.train_prior, self.natural_prior)[0]
134
+ best = int(probs.argmax())
135
+ result = (LABELS[best], float(probs[best]))
136
+ self._cache[key] = result
137
+ if len(self._cache) > CACHE_SIZE:
138
+ self._cache.popitem(last=False)
139
+ return result
140
+
141
+
142
+ def load_router(folder: str | None = None, repo: str | None = None) -> OnnxRouter | None:
143
+ """The router from a local folder, else from the Hugging Face repo; None if unavailable
144
+ (the gateway then adds no briefs: fail open)."""
145
+ try:
146
+ path = Path(folder).expanduser() if folder else None
147
+ if path is None or not all((path / f).exists() for f in MODEL_FILES):
148
+ from huggingface_hub import snapshot_download
149
+
150
+ path = Path(snapshot_download(repo_id=repo or DEFAULT_REPO, allow_patterns=list(MODEL_FILES),
151
+ local_dir=str(DEFAULT_CACHE)))
152
+ router = OnnxRouter(path)
153
+ log.info("prompt router loaded from %s", path)
154
+ return router
155
+ except Exception:
156
+ log.warning("prompt router unavailable; no task briefs", exc_info=True)
157
+ return None
@@ -0,0 +1,91 @@
1
+ """Trim long command output (test runs, builds) and search results.
2
+
3
+ Keeps the lines that matter most for fixing code, in this priority order, until the token
4
+ budget is used: the end of the output (test summaries, final error), lines that look like
5
+ errors with a little context, then the beginning. Every dropped run becomes one marker.
6
+ """
7
+
8
+ import re
9
+
10
+ from tokencut.tokens import count_tokens
11
+
12
+ _ERROR = re.compile(
13
+ r"error|exception|traceback|failed|failure|assert|fatal|panic|warning|"
14
+ r"\bE\s{2,}|^\s*File \"|^FAIL|^ERROR",
15
+ re.IGNORECASE,
16
+ )
17
+ TAIL_LINES = 40
18
+ HEAD_LINES = 15
19
+ ERROR_CONTEXT = 2 # lines kept before and after an error line
20
+ # Tables (CSV / TSV / Markdown) are never cut with head / tail: the answer can sit in any
21
+ # middle row (docs/ATTACHMENTS_PLAN.md, step 2).
22
+ TABLE_SAMPLE_LINES = 40
23
+ TABLE_MIN_LINES = 5
24
+ TABLE_SHARE = 0.8 # share of sampled lines with the same number of delimiters
25
+ _TABLE_DELIMITERS = (",", "\t", "|", ";")
26
+ _READ_PREFIX = re.compile(r"^\s*\d+→") # Claude Code's Read line numbers
27
+
28
+
29
+ def looks_like_table(text: str) -> bool:
30
+ """Most lines carry the same number (>= 2) of one delimiter: CSV, TSV, Markdown table."""
31
+ lines = [_READ_PREFIX.sub("", line, count=1) for line in text.split("\n")]
32
+ lines = [line for line in lines if line.strip()][:TABLE_SAMPLE_LINES]
33
+ if len(lines) < TABLE_MIN_LINES:
34
+ return False
35
+ for delimiter in _TABLE_DELIMITERS:
36
+ counts = [line.count(delimiter) for line in lines]
37
+ common = max(sorted(set(counts)), key=counts.count)
38
+ if common >= 2 and counts.count(common) >= TABLE_SHARE * len(lines):
39
+ return True
40
+ return False
41
+
42
+
43
+ def _priority_order(lines: list[str]) -> list[int]:
44
+ """Line indexes in the order we want to keep them (no duplicates)."""
45
+ order: list[int] = []
46
+ seen: set[int] = set()
47
+
48
+ def push(i: int) -> None:
49
+ if 0 <= i < len(lines) and i not in seen:
50
+ seen.add(i)
51
+ order.append(i)
52
+
53
+ for i in range(len(lines) - 1, max(-1, len(lines) - 1 - TAIL_LINES), -1):
54
+ push(i)
55
+ for i, line in enumerate(lines):
56
+ if _ERROR.search(line):
57
+ for k in range(i - ERROR_CONTEXT, i + ERROR_CONTEXT + 1):
58
+ push(k)
59
+ for i in range(min(HEAD_LINES, len(lines))):
60
+ push(i)
61
+ return order
62
+
63
+
64
+ def trim_output(text: str, budget_tokens: int, what: str = "output") -> str | None:
65
+ """Trimmed text within roughly budget_tokens, or None if no trim is needed/possible."""
66
+ if count_tokens(text) <= budget_tokens:
67
+ return None
68
+ lines = text.split("\n")
69
+ keep: set[int] = set()
70
+ used = 0
71
+ for i in _priority_order(lines):
72
+ cost = count_tokens(lines[i]) + 1
73
+ if used + cost > budget_tokens:
74
+ continue # a shorter line further down the priority list may still fit
75
+ keep.add(i)
76
+ used += cost
77
+
78
+ out: list[str] = []
79
+ i = 0
80
+ while i < len(lines):
81
+ if i in keep:
82
+ out.append(lines[i])
83
+ i += 1
84
+ continue
85
+ j = i
86
+ while j < len(lines) and j not in keep:
87
+ j += 1
88
+ out.append(f"[TokenCut: {j - i} lines of {what} omitted]")
89
+ i = j
90
+ result = "\n".join(out)
91
+ return result if count_tokens(result) < count_tokens(text) else None