tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Relevance of every chunk to the query (IMPLEMENTATION_PLAN C2).
|
|
2
|
+
|
|
3
|
+
score = ALPHA * structural_score + (1 - ALPHA) * text_score
|
|
4
|
+
text_score BM25 over identifier sub-words, normalised to [0, 1]
|
|
5
|
+
structural_score weight / (1 + graph distance) from the chunks the query mentions
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
from rank_bm25 import BM25Okapi
|
|
11
|
+
|
|
12
|
+
from tokencut.contracts import Chunk, CodeGraph, ScoredChunk
|
|
13
|
+
from tokencut.graph.queries import GraphIndex
|
|
14
|
+
from tokencut.trimming.query import SYMBOL_WEIGHT, QueryMentions, analyse_query
|
|
15
|
+
|
|
16
|
+
ALPHA = 0.5 # (tunable) log changes in the DECISIONS.md change log
|
|
17
|
+
PSEUDO_MENTIONS = 3 # fallback when the query names nothing: top-N chunks by text score
|
|
18
|
+
|
|
19
|
+
_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
20
|
+
_CAMEL_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|[0-9]+")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def split_identifier(identifier: str) -> list[str]:
|
|
24
|
+
"""'apply_discount' -> ['apply', 'discount']; 'HTTPServerError' -> ['http', 'server', 'error']."""
|
|
25
|
+
words = []
|
|
26
|
+
for part in identifier.split("_"):
|
|
27
|
+
for word in _CAMEL_RE.findall(part):
|
|
28
|
+
if len(word) >= 2:
|
|
29
|
+
words.append(word.lower())
|
|
30
|
+
return words
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def tokenize(text: str) -> list[str]:
|
|
34
|
+
words = []
|
|
35
|
+
for identifier in _IDENT_RE.findall(text):
|
|
36
|
+
words.extend(split_identifier(identifier))
|
|
37
|
+
return words
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _bm25_normalised(documents: list[list[str]], query: str) -> list[float]:
|
|
41
|
+
"""BM25 score of each document, divided by the max (all zeros if nothing matches)."""
|
|
42
|
+
query_words = tokenize(query)
|
|
43
|
+
if not query_words or not any(documents):
|
|
44
|
+
return [0.0] * len(documents)
|
|
45
|
+
raw = BM25Okapi(documents).get_scores(query_words)
|
|
46
|
+
clipped = [max(0.0, float(s)) for s in raw]
|
|
47
|
+
best = max(clipped, default=0.0)
|
|
48
|
+
if best <= 0:
|
|
49
|
+
return [0.0] * len(documents)
|
|
50
|
+
return [s / best for s in clipped]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def text_scores(chunks: list[Chunk], query: str) -> list[float]:
|
|
54
|
+
"""BM25 score of each chunk, normalised to [0, 1]."""
|
|
55
|
+
documents = [tokenize(f"{c.file_path} {c.qualified_name} {c.code}") for c in chunks]
|
|
56
|
+
return _bm25_normalised(documents, query)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def file_text_scores(chunks: list[Chunk], query: str) -> dict[str, float]:
|
|
60
|
+
"""BM25 score of each whole file (path + all its code), normalised to [0, 1].
|
|
61
|
+
|
|
62
|
+
Used to rank files for the evaluation pool: a file whose words match the issue as a
|
|
63
|
+
whole can matter even when no single chunk scores high.
|
|
64
|
+
"""
|
|
65
|
+
files = sorted({c.file_path for c in chunks})
|
|
66
|
+
words: dict[str, list[str]] = {f: tokenize(f) for f in files}
|
|
67
|
+
for c in chunks:
|
|
68
|
+
words[c.file_path].extend(tokenize(c.code))
|
|
69
|
+
return dict(zip(files, _bm25_normalised([words[f] for f in files], query)))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def structural_scores(index: GraphIndex, weights: dict[str, float]) -> dict[str, float]:
|
|
73
|
+
"""For each reachable chunk: max over mentioned chunks of weight / (1 + distance)."""
|
|
74
|
+
result: dict[str, float] = {}
|
|
75
|
+
for weight in sorted(set(weights.values()), reverse=True):
|
|
76
|
+
sources = sorted(cid for cid, w in weights.items() if w == weight)
|
|
77
|
+
for cid, distance in index.distances(sources).items():
|
|
78
|
+
result[cid] = max(result.get(cid, 0.0), weight / (1 + distance))
|
|
79
|
+
return result
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def score_chunks(
|
|
83
|
+
chunks: list[Chunk],
|
|
84
|
+
graph: CodeGraph,
|
|
85
|
+
query: str,
|
|
86
|
+
mentions: QueryMentions | None = None,
|
|
87
|
+
index: GraphIndex | None = None,
|
|
88
|
+
) -> list[ScoredChunk]:
|
|
89
|
+
"""Score every chunk's relevance to the query (BM25 text score + graph distance)."""
|
|
90
|
+
if mentions is None:
|
|
91
|
+
mentions = analyse_query(query, chunks)
|
|
92
|
+
if index is None:
|
|
93
|
+
index = GraphIndex(graph)
|
|
94
|
+
|
|
95
|
+
texts = text_scores(chunks, query)
|
|
96
|
+
text_by_id = {c.id: t for c, t in zip(chunks, texts)}
|
|
97
|
+
|
|
98
|
+
weights = dict(mentions.weights)
|
|
99
|
+
if not weights:
|
|
100
|
+
# Fallback: the query names nothing, so the best text matches act as the mentions.
|
|
101
|
+
ranked = sorted(
|
|
102
|
+
(cid for cid, t in text_by_id.items() if t > 0), key=lambda cid: (-text_by_id[cid], cid)
|
|
103
|
+
)
|
|
104
|
+
weights = {cid: SYMBOL_WEIGHT for cid in ranked[:PSEUDO_MENTIONS]}
|
|
105
|
+
structural = structural_scores(index, weights)
|
|
106
|
+
|
|
107
|
+
scored = []
|
|
108
|
+
for c in chunks:
|
|
109
|
+
s = structural.get(c.id, 0.0)
|
|
110
|
+
t = text_by_id[c.id]
|
|
111
|
+
scored.append(
|
|
112
|
+
ScoredChunk(
|
|
113
|
+
chunk_id=c.id, score=ALPHA * s + (1 - ALPHA) * t, structural_score=s, text_score=t
|
|
114
|
+
)
|
|
115
|
+
)
|
|
116
|
+
scored.sort(key=lambda sc: (-sc.score, sc.chunk_id)) # DECISIONS #8: ties by chunk id
|
|
117
|
+
return scored
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Member C — Context scoring and 4-tier variant generation.
|
|
2
|
+
|
|
3
|
+
Note: works on chunks + graph, NOT on a repo path, so the same code can later
|
|
4
|
+
trim chunks cut out of an agent's request inside the gateway.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from tokencut.contracts import AGGRESSIVE_MIN_TOKENS, Chunk, CodeGraph, ContextVariant, ScoredChunk
|
|
8
|
+
from tokencut.graph.queries import GraphIndex
|
|
9
|
+
from tokencut.tokens import count_tokens
|
|
10
|
+
from tokencut.trimming import scoring
|
|
11
|
+
from tokencut.trimming.packing import TierPacker, build_pool, tier_budgets
|
|
12
|
+
from tokencut.trimming.query import analyse_query
|
|
13
|
+
from tokencut.trimming.render import render_variant
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def score_chunks(chunks: list[Chunk], graph: CodeGraph, query: str) -> list[ScoredChunk]:
|
|
17
|
+
"""Score every chunk's relevance to the query (BM25 text score + graph distance)."""
|
|
18
|
+
return scoring.score_chunks(chunks, graph, query)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def generate_variants(
|
|
22
|
+
chunks: list[Chunk],
|
|
23
|
+
graph: CodeGraph,
|
|
24
|
+
query: str,
|
|
25
|
+
full_cap_tokens: int | None = None,
|
|
26
|
+
aggressive_min_tokens: int = AGGRESSIVE_MIN_TOKENS,
|
|
27
|
+
) -> list[ContextVariant]:
|
|
28
|
+
"""Return exactly 4 variants (aggressive, moderate, conservative, full).
|
|
29
|
+
|
|
30
|
+
Must:
|
|
31
|
+
- respect each tier's token budget
|
|
32
|
+
- have token counts that never decrease from aggressive -> full
|
|
33
|
+
- be deterministic (break score ties by chunk id)
|
|
34
|
+
|
|
35
|
+
full_cap_tokens: None = gateway mode (everything is "full"); a number = evaluation mode
|
|
36
|
+
(top-ranked files up to that many tokens, e.g. EVAL_FULL_CONTEXT_CAP_TOKENS).
|
|
37
|
+
"""
|
|
38
|
+
chunks = sorted(chunks, key=lambda c: (c.file_path, c.start_line, c.id))
|
|
39
|
+
index = GraphIndex(graph)
|
|
40
|
+
mentions = analyse_query(query, chunks)
|
|
41
|
+
scores = scoring.score_chunks(chunks, graph, query, mentions=mentions, index=index)
|
|
42
|
+
|
|
43
|
+
file_text = scoring.file_text_scores(chunks, query) if full_cap_tokens is not None else None
|
|
44
|
+
pool = build_pool(chunks, scores, full_cap_tokens, file_text)
|
|
45
|
+
full_tokens = count_tokens(render_variant(pool, {c.id for c in pool}, set()))
|
|
46
|
+
budgets = tier_budgets(full_tokens, aggressive_min_tokens)
|
|
47
|
+
|
|
48
|
+
order = {c.id: (c.file_path, c.start_line, c.id) for c in pool}
|
|
49
|
+
variants = []
|
|
50
|
+
for packed in TierPacker(pool, scores, index).pack(budgets):
|
|
51
|
+
variants.append(
|
|
52
|
+
ContextVariant(
|
|
53
|
+
tier=packed.tier,
|
|
54
|
+
budget_tokens=packed.budget_tokens,
|
|
55
|
+
kept_chunk_ids=sorted(packed.kept, key=order.__getitem__),
|
|
56
|
+
skeleton_chunk_ids=sorted(packed.skeletons, key=order.__getitem__),
|
|
57
|
+
token_count=packed.token_count,
|
|
58
|
+
rendered_text=packed.rendered_text,
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
return variants
|
tokencut/userconfig.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""The installed tool's settings: ~/.tokencut/config.json (no secrets) and two private keys.
|
|
2
|
+
|
|
3
|
+
`tokencut setup` writes the file; `tokencut` (tokencut.launch) turns it into the gateway's
|
|
4
|
+
TOKENCUT_* environment. A TOKENCUT_* variable the user already set always wins, so every
|
|
5
|
+
gateway setting in CLAUDE.md / README keeps working. TOKENCUT_HOME moves the folder (tests).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import copy
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import secrets
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
DEFAULTS: dict = {
|
|
15
|
+
"version": 1,
|
|
16
|
+
"gateway": {
|
|
17
|
+
"port": 8787,
|
|
18
|
+
"mode": "trim",
|
|
19
|
+
"tier": "moderate",
|
|
20
|
+
"system_note": True,
|
|
21
|
+
"prompt_brief": True,
|
|
22
|
+
},
|
|
23
|
+
"litellm": {"enabled": False, "port": 8788}, # model connector; Claude + GPT go through it
|
|
24
|
+
"chatgpt": {"enabled": False, "models": ["gpt-6.1-sol", "gpt-6-luna", "gpt-6-astra"]},
|
|
25
|
+
}
|
|
26
|
+
KEY_FILES = {"tokencut": "tokencut_key", "litellm": "litellm_master_key"}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def home() -> Path:
|
|
30
|
+
return Path(os.environ.get("TOKENCUT_HOME") or Path.home() / ".tokencut")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def config_path() -> Path:
|
|
34
|
+
return home() / "config.json"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _merge(base: dict, extra: dict) -> dict:
|
|
38
|
+
out = copy.deepcopy(base)
|
|
39
|
+
for key, value in extra.items():
|
|
40
|
+
out[key] = _merge(out[key], value) if isinstance(value, dict) and isinstance(
|
|
41
|
+
out.get(key), dict) else value
|
|
42
|
+
return out
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def load() -> dict:
|
|
46
|
+
"""The saved settings on top of the defaults (defaults alone if there is no file)."""
|
|
47
|
+
path = config_path()
|
|
48
|
+
saved = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
49
|
+
return _merge(DEFAULTS, saved)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def save(config: dict) -> Path:
|
|
53
|
+
path = config_path()
|
|
54
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
tmp = path.with_suffix(".tmp")
|
|
56
|
+
tmp.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
57
|
+
tmp.replace(path)
|
|
58
|
+
return path
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def key(name: str, create: bool = True) -> str | None:
|
|
62
|
+
"""A private random key from ~/.tokencut (owner-only file); created on first use."""
|
|
63
|
+
path = home() / KEY_FILES[name]
|
|
64
|
+
if path.exists():
|
|
65
|
+
return path.read_text(encoding="utf-8").strip() or None
|
|
66
|
+
if not create:
|
|
67
|
+
return None
|
|
68
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
value = ("tc-" if name == "tokencut" else "sk-") + secrets.token_hex(32)
|
|
70
|
+
path.write_text(value, encoding="utf-8")
|
|
71
|
+
path.chmod(0o600)
|
|
72
|
+
return value
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def gateway_env(config: dict, litellm_url: str | None, environ: dict[str, str]) -> dict[str, str]:
|
|
76
|
+
"""Environment for the gateway process: config values, never overriding the user's own."""
|
|
77
|
+
gw = config["gateway"]
|
|
78
|
+
wanted = {
|
|
79
|
+
"TOKENCUT_PORT": str(gw["port"]),
|
|
80
|
+
"TOKENCUT_MODE": gw["mode"],
|
|
81
|
+
"TOKENCUT_TIER": gw["tier"],
|
|
82
|
+
"TOKENCUT_SYSTEM_NOTE": "1" if gw["system_note"] else "0",
|
|
83
|
+
"TOKENCUT_PROMPT_BRIEF": "1" if gw["prompt_brief"] else "0",
|
|
84
|
+
"TOKENCUT_LOG_DIR": str(home() / "logs"),
|
|
85
|
+
}
|
|
86
|
+
tokencut_key = key("tokencut", create=False)
|
|
87
|
+
if tokencut_key:
|
|
88
|
+
wanted["TOKENCUT_KEY"] = tokencut_key
|
|
89
|
+
if litellm_url:
|
|
90
|
+
wanted["TOKENCUT_GPT_UPSTREAM"] = litellm_url
|
|
91
|
+
wanted["TOKENCUT_CLAUDE_VIA_LITELLM"] = "1"
|
|
92
|
+
litellm_key = key("litellm", create=False)
|
|
93
|
+
if litellm_key:
|
|
94
|
+
wanted["TOKENCUT_GPT_KEY"] = litellm_key
|
|
95
|
+
return {**wanted, **environ}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def agent_env(config: dict, environ: dict[str, str]) -> dict[str, str]:
|
|
99
|
+
"""Environment for Claude Code: talk to the gateway and send the TokenCut key."""
|
|
100
|
+
port = environ.get("TOKENCUT_PORT", str(config["gateway"]["port"]))
|
|
101
|
+
out = {**environ, "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}/anthropic"}
|
|
102
|
+
tokencut_key = key("tokencut", create=False)
|
|
103
|
+
if tokencut_key:
|
|
104
|
+
headers = [line for line in environ.get("ANTHROPIC_CUSTOM_HEADERS", "").splitlines()
|
|
105
|
+
if line.strip() and not line.lower().startswith("x-tokencut-key:")]
|
|
106
|
+
out["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join([*headers, f"x-tokencut-key: {tokencut_key}"])
|
|
107
|
+
return out
|
tokencut/wizard.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""`tokencut setup`: first-run questions and one-time installs.
|
|
2
|
+
|
|
3
|
+
1. Claude Code must be installed (the user installs it; we print the official command).
|
|
4
|
+
2. Ask whether to add the ChatGPT plan (GPT models in Claude Code via `/model gpt-…`).
|
|
5
|
+
3. Create the private keys, install the hidden model connector (LiteLLM), sign in to ChatGPT.
|
|
6
|
+
4. Download the prompt router model once (task briefs), so the first start is fast.
|
|
7
|
+
5. Save ~/.tokencut/config.json.
|
|
8
|
+
Claude itself needs no step here: Claude Code asks for its own login on first start, and that
|
|
9
|
+
login travels through TokenCut unchanged. Every failure leaves a working Claude-only setup.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import shutil
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
|
|
16
|
+
from tokencut import litellm_proxy, userconfig
|
|
17
|
+
|
|
18
|
+
CLAUDE_INSTALL = {
|
|
19
|
+
"posix": "curl -fsSL https://claude.ai/install.sh | bash",
|
|
20
|
+
"nt": "irm https://claude.ai/install.ps1 | iex",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _yes(answer: str, default: bool) -> bool:
|
|
25
|
+
answer = answer.strip().lower()
|
|
26
|
+
return default if not answer else answer in ("y", "yes")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _download_router(out: Callable[[str], None]) -> bool:
|
|
30
|
+
from tokencut.gateway.task_router import load_router
|
|
31
|
+
|
|
32
|
+
out("Downloading the prompt model (about 135 MB, one time) ...")
|
|
33
|
+
return load_router() is not None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def run_setup(
|
|
37
|
+
chatgpt: bool | None = None,
|
|
38
|
+
ask: Callable[[str], str] = input,
|
|
39
|
+
out: Callable[[str], None] = print,
|
|
40
|
+
which: Callable[[str], str | None] = shutil.which,
|
|
41
|
+
os_name: str | None = None,
|
|
42
|
+
install_connector: Callable[[], bool] = litellm_proxy.install,
|
|
43
|
+
connector_ready: Callable[[], bool] = litellm_proxy.installed,
|
|
44
|
+
chatgpt_ready: Callable[[], bool] = litellm_proxy.chatgpt_logged_in,
|
|
45
|
+
chatgpt_login: Callable[[], bool] = litellm_proxy.chatgpt_login,
|
|
46
|
+
download_router: Callable[[Callable[[str], None]], bool] = _download_router,
|
|
47
|
+
) -> int:
|
|
48
|
+
if which("claude") is None:
|
|
49
|
+
out("TokenCut works inside Claude Code, which is not installed yet. Install it with:")
|
|
50
|
+
out(f" {CLAUDE_INSTALL.get(os_name or os.name, CLAUDE_INSTALL['posix'])}")
|
|
51
|
+
out("then run `tokencut setup` again.")
|
|
52
|
+
return 1
|
|
53
|
+
config = userconfig.load()
|
|
54
|
+
if chatgpt is None:
|
|
55
|
+
chatgpt = _yes(ask("Also use your ChatGPT plan for GPT models? [y/N] "), False)
|
|
56
|
+
|
|
57
|
+
userconfig.key("tokencut")
|
|
58
|
+
userconfig.key("litellm")
|
|
59
|
+
|
|
60
|
+
ready = connector_ready()
|
|
61
|
+
if not ready:
|
|
62
|
+
out("Installing the model connector (one time, 1-3 minutes) ...")
|
|
63
|
+
ready = install_connector()
|
|
64
|
+
if not ready:
|
|
65
|
+
out("Could not install the model connector; Claude will connect directly, "
|
|
66
|
+
"GPT models are off. Run `tokencut setup` again to retry.")
|
|
67
|
+
config["litellm"]["enabled"] = ready
|
|
68
|
+
|
|
69
|
+
config["chatgpt"]["enabled"] = False
|
|
70
|
+
if chatgpt and ready:
|
|
71
|
+
if not chatgpt_ready():
|
|
72
|
+
out("ChatGPT sign-in: open the link below, sign in, and enter the code.")
|
|
73
|
+
if not chatgpt_login():
|
|
74
|
+
out("ChatGPT sign-in did not finish; GPT models are off for now.")
|
|
75
|
+
config["chatgpt"]["enabled"] = chatgpt_ready()
|
|
76
|
+
elif chatgpt:
|
|
77
|
+
out("GPT models need the model connector; they stay off for now.")
|
|
78
|
+
|
|
79
|
+
if config["gateway"]["prompt_brief"] and not download_router(out):
|
|
80
|
+
out("Prompt model unavailable; task briefs are off (everything else works).")
|
|
81
|
+
config["gateway"]["prompt_brief"] = False
|
|
82
|
+
|
|
83
|
+
if ready:
|
|
84
|
+
litellm_proxy.write_config(config)
|
|
85
|
+
path = userconfig.save(config)
|
|
86
|
+
models = "Claude" + (" + ChatGPT (" + ", ".join(config["chatgpt"]["models"]) + ")"
|
|
87
|
+
if config["chatgpt"]["enabled"] else "")
|
|
88
|
+
out(f"\nTokenCut is ready ({path}).")
|
|
89
|
+
out(f" Models: {models}")
|
|
90
|
+
out(f" Trimming: {config['gateway']['mode']} ({config['gateway']['tier']}), "
|
|
91
|
+
f"system note: {'on' if config['gateway']['system_note'] else 'off'}, "
|
|
92
|
+
f"task briefs: {'on' if config['gateway']['prompt_brief'] else 'off'}")
|
|
93
|
+
out("Run `tokencut` in your project folder to open Claude Code through TokenCut.")
|
|
94
|
+
return 0
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tokencut-proxy
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cache-aware gateway that trims the context coding agents send to the model
|
|
5
|
+
Author: Masooma Ali, Huma Taj, Fatima Shakir
|
|
6
|
+
Project-URL: Repository, https://github.com/Masooma-Ali/TokenCut
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: tree-sitter<0.27,>=0.26
|
|
10
|
+
Requires-Dist: tree-sitter-python<0.26,>=0.25
|
|
11
|
+
Requires-Dist: networkx<4,>=3.6
|
|
12
|
+
Requires-Dist: pydantic<3,>=2.13
|
|
13
|
+
Requires-Dist: tiktoken<1,>=0.14
|
|
14
|
+
Requires-Dist: rank-bm25<0.3,>=0.2.2
|
|
15
|
+
Requires-Dist: fastapi<1,>=0.142
|
|
16
|
+
Requires-Dist: uvicorn<1,>=0.53
|
|
17
|
+
Requires-Dist: httpx2<3,>=2.13
|
|
18
|
+
Requires-Dist: pillow<14,>=13
|
|
19
|
+
Requires-Dist: pypdf<7,>=6.21
|
|
20
|
+
Requires-Dist: numpy<3,>=2.4
|
|
21
|
+
Requires-Dist: onnxruntime<2,>=1.30
|
|
22
|
+
Requires-Dist: tokenizers<1,>=0.23
|
|
23
|
+
Requires-Dist: huggingface-hub<2,>=1.33
|
|
24
|
+
Provides-Extra: embed
|
|
25
|
+
Requires-Dist: fastembed<1,>=0.9.1; extra == "embed"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: fastembed==0.9.1; extra == "dev"
|
|
28
|
+
Requires-Dist: unidiff==1.0.1; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest==9.1.1; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff==0.16.10; extra == "dev"
|
|
31
|
+
|
|
32
|
+
# TokenCut
|
|
33
|
+
|
|
34
|
+
Cost-aware joint context-and-model routing gateway for AI coding agents (FYP, FAST NUCES Islamabad).
|
|
35
|
+
|
|
36
|
+
**Iteration 1:** repository parsing, code-graph construction, and 4-tier context-variant generation.
|
|
37
|
+
|
|
38
|
+
## Install and use (one command)
|
|
39
|
+
Needs [Claude Code](https://claude.ai/code) installed first. Then:
|
|
40
|
+
```bash
|
|
41
|
+
curl -fsSL https://github.com/Masooma-Ali/TokenCut/releases/latest/download/install.sh | sh # macOS / Linux
|
|
42
|
+
```
|
|
43
|
+
```powershell
|
|
44
|
+
irm https://github.com/Masooma-Ali/TokenCut/releases/latest/download/install.ps1 | iex # Windows
|
|
45
|
+
```
|
|
46
|
+
The installer adds [uv](https://docs.astral.sh/uv/) if missing, installs the `tokencut` command
|
|
47
|
+
(PyPI package `tokencut-proxy`) and runs `tokencut setup`, which asks whether to add your ChatGPT
|
|
48
|
+
plan, installs the hidden model connector (LiteLLM) and downloads the prompt model once.
|
|
49
|
+
Then, in any project folder:
|
|
50
|
+
|
|
51
|
+
| Command | What it does |
|
|
52
|
+
|---|---|
|
|
53
|
+
| `tokencut` | opens Claude Code through TokenCut (trimming, system note, task briefs, routing); `/model gpt-6.1-sol` for GPT if set up |
|
|
54
|
+
| `tokencut setup` | change logins / re-run one-time installs (`--chatgpt`, `--no-chatgpt`) |
|
|
55
|
+
| `tokencut status` | running? tokens removed, billed usage |
|
|
56
|
+
| `tokencut logs [gateway\|connector]` | last lines of a process log |
|
|
57
|
+
|
|
58
|
+
Settings: `~/.tokencut/config.json` (no secrets); every `TOKENCUT_*` variable below overrides it.
|
|
59
|
+
Claude keeps Claude Code's own login; the ChatGPT token stays in LiteLLM's file; a local
|
|
60
|
+
TokenCut key (`~/.tokencut/tokencut_key`) stops other programs from using the gateway.
|
|
61
|
+
Releases: push a tag `vX.Y.Z` (`.github/workflows/release.yml` tests, publishes to PyPI and
|
|
62
|
+
creates a GitHub Release with the install scripts attached; the `releases/latest/download/…`
|
|
63
|
+
links above always point to the newest one). The links work only while the repository is public.
|
|
64
|
+
|
|
65
|
+
## Pipeline
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
repo ──► ingestion.parse_repo() ──► list[Chunk]
|
|
69
|
+
│
|
|
70
|
+
▼
|
|
71
|
+
graph.build_graph() ──► CodeGraph
|
|
72
|
+
│
|
|
73
|
+
query ───────────────►▼
|
|
74
|
+
trimming.generate_variants() ──► 4 × ContextVariant
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
All shared data formats live in `tokencut/contracts.py`. Changing that file needs a PR approved by both other members.
|
|
78
|
+
|
|
79
|
+
## Usage
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from tokencut.ingestion.parser import parse_repo
|
|
83
|
+
from tokencut.graph.builder import build_graph
|
|
84
|
+
from tokencut.trimming.variants import generate_variants
|
|
85
|
+
|
|
86
|
+
chunks = parse_repo("fixtures/sample_repo")
|
|
87
|
+
graph = build_graph(chunks)
|
|
88
|
+
for v in generate_variants(chunks, graph, "Cart.total applies the discount wrong"):
|
|
89
|
+
print(v.tier.value, v.token_count, "/", v.budget_tokens)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
API: `uvicorn api.main:app --reload`, then `POST /context/variants` with
|
|
93
|
+
`{"repo_path": "...", "query": "...", "include_text": true}`.
|
|
94
|
+
|
|
95
|
+
Evaluation on SWE-bench Lite (clones repos into `swebench_repos/`, caches in `cache/`):
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
python -m eval.run_eval --limit 3 # smoke run
|
|
99
|
+
python -m eval.run_eval # all tasks in eval/tasks_iter1.json
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Results go to `eval/results/iter1_results.csv` and `eval/results/iter1_summary.md`.
|
|
103
|
+
|
|
104
|
+
## Benchmark: TokenCut vs baselines
|
|
105
|
+
|
|
106
|
+
`eval/benchmark.py` compares TokenCut with BM25, dense embeddings (bge-small), naive
|
|
107
|
+
truncation, random and SWE-bench file retrieval at **equal token budgets** (15 / 35 / 60 %
|
|
108
|
+
of a 32k context), in two settings: pure trimming of a shared neutral 32k context, and
|
|
109
|
+
selection from the whole repository. Reports gold-chunk recall with 95% bootstrap CIs and
|
|
110
|
+
paired sign tests. Tasks: 40 dev (used for tuning) and 81 held-out test tasks
|
|
111
|
+
(`eval/tasks_iter2.json`).
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
python -m eval.prepare --set test # clone + parse once
|
|
115
|
+
python -m eval.benchmark --set test # no embeddings
|
|
116
|
+
python -m eval.benchmark --set test --methods tokencut,tokencut_hybrid,bm25,embed,random,head,bm25_files
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Results: `eval/results/benchmark_<set>.md` / `.csv`.
|
|
120
|
+
|
|
121
|
+
## Gateway (Iteration 2)
|
|
122
|
+
|
|
123
|
+
The gateway sits between your coding agent and the provider. It streams the reply back and
|
|
124
|
+
records what each request contained (tokens per kind: system prompt, tools, chat, file
|
|
125
|
+
reads, command output…) plus the tokens the provider actually billed, including cached
|
|
126
|
+
tokens. In shadow mode (default) requests are forwarded unchanged and the stats show what
|
|
127
|
+
trimming *would* save; in trim mode the trimmed request is sent.
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
python -m tokencut.gateway # listens on 127.0.0.1:8787
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Point an agent at it:
|
|
134
|
+
|
|
135
|
+
| Agent / SDK | Setting |
|
|
136
|
+
|---|---|
|
|
137
|
+
| Claude Code (Anthropic) | `ANTHROPIC_BASE_URL=http://127.0.0.1:8787/anthropic` |
|
|
138
|
+
| OpenAI SDKs, Cline, OpenRouter-style clients | base URL `http://127.0.0.1:8787/openai/v1` |
|
|
139
|
+
| Gemini (google-genai / Gemini CLI) | base URL `http://127.0.0.1:8787/gemini` |
|
|
140
|
+
|
|
141
|
+
Your API key travels in the request headers to the provider as usual; it is never logged.
|
|
142
|
+
See what was measured at `http://127.0.0.1:8787/tokencut/stats`; per-request numbers go to
|
|
143
|
+
`logs/gateway/requests-<date>.jsonl` (gitignored).
|
|
144
|
+
|
|
145
|
+
| Variable | Default | Meaning |
|
|
146
|
+
|---|---|---|
|
|
147
|
+
| `TOKENCUT_ANTHROPIC_UPSTREAM` / `_OPENAI_` / `_GEMINI_` | official API URLs | send to another compatible endpoint (e.g. OpenRouter) |
|
|
148
|
+
| `TOKENCUT_LOG_DIR` | `logs/gateway` | where numbers are logged (empty = off) |
|
|
149
|
+
| `TOKENCUT_KEY` | none | if set, proxy requests must send it in `x-tokencut-key` (the `tokencut` command does this; the header is never forwarded) |
|
|
150
|
+
| `TOKENCUT_CAPTURE` | off | `1` = also save full request bodies locally, for parser test fixtures (contain your prompts and code) |
|
|
151
|
+
| `TOKENCUT_HOST` / `TOKENCUT_PORT` | `127.0.0.1` / `8787` | listen address |
|
|
152
|
+
| `TOKENCUT_MODE` | `shadow` | `shadow` = forward unchanged, report possible savings; `trim` = send the trimmed request |
|
|
153
|
+
| `TOKENCUT_TIER` | `moderate` | `aggressive` / `moderate` / `conservative` |
|
|
154
|
+
| `TOKENCUT_MIN_TRIM_TOKENS` | `2000` | tool results smaller than this are never trimmed |
|
|
155
|
+
| `TOKENCUT_CACHE_EVENTS` | `1` | `0` = never re-trim history at cache events |
|
|
156
|
+
| `TOKENCUT_CACHE_TTL` | `300` | seconds of idle after which the provider cache counts as expired |
|
|
157
|
+
| `TOKENCUT_PRUNE_TOOLS` | off | `1` = at cache events, send only tools used so far + core tools |
|
|
158
|
+
| `TOKENCUT_CORE_TOOLS` | Claude Code / Gemini CLI basics | comma-separated tools always kept when pruning |
|
|
159
|
+
| `TOKENCUT_TASK_MASK` | **on** | when the user starts a new task, earlier tasks' large tool outputs become notes (only if the cache rewrite pays off); `0` = off |
|
|
160
|
+
| `TOKENCUT_TASK_MASK_TOKENS` | `300` | smallest tool output masked at a new task |
|
|
161
|
+
| `TOKENCUT_CCR` | off | `1` = every shortened output gets a reference; the gateway offers `tokencut_retrieve` and answers it with the exact original lines |
|
|
162
|
+
| `TOKENCUT_LAZY_TOOLS` | off | `1` = every request carries only core / used tools + a catalog tool `tokencut_enable_tools` for the rest |
|
|
163
|
+
| `TOKENCUT_SYSTEM_NOTE` | off | `1` = trim mode adds a fixed ~125-token note to the system prompt: how to treat `[TokenCut: …]` markers (re-read only the omitted lines, do not re-run commands for cut log lines, keep test output short) |
|
|
164
|
+
| `TOKENCUT_PROMPT_BRIEF` | off | `1` = trim mode classifies each typed user message (rules + the ONNX prompt router) and, when confident, appends a short task brief (`<tokencut_task_brief type="bugfix">…`, 48–71 tokens); agent requests only; never repeated for the same type in a row |
|
|
165
|
+
| `TOKENCUT_BRIEF_THRESHOLD` | `0.87` | minimum router probability for a brief (chosen on dev, RESULTS_REPORT §11) |
|
|
166
|
+
| `TOKENCUT_ROUTER_DIR` | — | local folder with `router.onnx`, `router.onnx.data`, `tokenizer.json`, `router_config.json` |
|
|
167
|
+
| `TOKENCUT_ROUTER_REPO` | `humataj/tokencut-router` | public Hugging Face repo the model is downloaded from (once, no login, to `~/.cache/tokencut/router`) when no local folder is set |
|
|
168
|
+
|
|
169
|
+
**Trimming (phase 3).** Only large tool results are shortened: Python file contents
|
|
170
|
+
(through the Iteration 1 engine; kept lines stay verbatim, gaps become
|
|
171
|
+
`[TokenCut: lines 40-95 of x.py omitted …]` markers so the model can re-read them),
|
|
172
|
+
command output (failures, errors and the summary are kept) and search results.
|
|
173
|
+
System prompt, tools, chat text and tool calls are never touched; files the user named or
|
|
174
|
+
the agent already edited stay whole. A result's trim depends only on earlier conversation,
|
|
175
|
+
so it is identical on every later turn and the provider's prompt cache keeps working.
|
|
176
|
+
|
|
177
|
+
**Cache-event trimming.** The cache is lost anyway when the model changes, after the cache
|
|
178
|
+
expires, or when the context is nearly full. At those moments TokenCut also shrinks old
|
|
179
|
+
history: old command/test/search output becomes a note with its error lines, older reads of
|
|
180
|
+
a file that was read again or edited become a short note, and (opt-in) unused tool
|
|
181
|
+
definitions are dropped. The last few turns are never touched. The decisions are remembered
|
|
182
|
+
per conversation and re-sent identically until the next event; a file the agent re-reads
|
|
183
|
+
after its old read was removed is protected from then on.
|
|
184
|
+
|
|
185
|
+
**Reversible trimming (P1, `TOKENCUT_CCR=1`).** Every shortened output ends with
|
|
186
|
+
`[TokenCut ref tc_…]`. The gateway adds a `tokencut_retrieve(ref, start_line, end_line)` tool;
|
|
187
|
+
the agent does not know it and answers "no such tool", and the gateway replaces that answer
|
|
188
|
+
with the exact original lines before forwarding. References are hashes of the original, so
|
|
189
|
+
they are identical every turn (cache-safe) and survive restarts.
|
|
190
|
+
|
|
191
|
+
**Lazy tool loading (P1, `TOKENCUT_LAZY_TOOLS=1`).** Each request carries only the core tools
|
|
192
|
+
(`TOKENCUT_CORE_TOOLS`), tools already used, and one catalog tool `tokencut_enable_tools`
|
|
193
|
+
listing the rest; enabling one adds it from that request on (one cache rewrite).
|
|
194
|
+
|
|
195
|
+
## Repo index for the agent (P1, local MCP server)
|
|
196
|
+
|
|
197
|
+
Not part of the gateway: a local MCP server that indexes the working tree (uncommitted
|
|
198
|
+
edits included) and gives the agent `repo_map`, `find_code` and `symbol`, so it can find
|
|
199
|
+
code in one call instead of several grep + read rounds.
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
python -m tokencut.repoindex build --repo . # first index (cached, incremental after)
|
|
203
|
+
claude mcp add tokencut -- python -m tokencut.repoindex serve --repo .
|
|
204
|
+
python -m tokencut.repoindex find --repo . "QuerySet.only() crashes with select_related"
|
|
205
|
+
```
|
|
206
|
+
`--embed` adds dense embeddings (computed once in the background, cached per chunk).
|
|
207
|
+
Offline check on SWE-bench tasks: `python -m eval.repoindex_eval --set dev|test`.
|
|
208
|
+
|
|
209
|
+
## Code map
|
|
210
|
+
|
|
211
|
+
| Module | What it does |
|
|
212
|
+
|---|---|
|
|
213
|
+
| `ingestion/walker.py` | file list (`git ls-files`, skip rules from DECISIONS #12) |
|
|
214
|
+
| `ingestion/python_parser.py` | tree-sitter chunker (module / class / function / method) |
|
|
215
|
+
| `graph/modules.py` | module names, import parsing, relative imports, `src/` layout |
|
|
216
|
+
| `graph/builder.py` | contains / imports / inherits / calls edges (DECISIONS #7) |
|
|
217
|
+
| `graph/queries.py` | `GraphIndex`: dependency closure, BFS distances |
|
|
218
|
+
| `trimming/query.py` | file / symbol / traceback mentions in the query |
|
|
219
|
+
| `trimming/scoring.py` | BM25 + graph-distance score |
|
|
220
|
+
| `trimming/packing.py` | full pool, tier budgets, nested greedy packing, skeletons |
|
|
221
|
+
| `trimming/render.py` | text sent to the model |
|
|
222
|
+
| `eval/loader.py`, `eval/metrics.py`, `eval/run_eval.py` | SWE-bench Lite subset, gold chunks, recall metrics |
|
|
223
|
+
| `gateway/app.py` | proxy server: forward, stream back, measure (shadow mode) |
|
|
224
|
+
| `gateway/adapters/` | Anthropic / OpenAI / Gemini request JSON → `Segment` list |
|
|
225
|
+
| `gateway/classify.py`, `gateway/usage.py`, `gateway/records.py` | tool-result kinds, billed usage from responses, request log |
|
|
226
|
+
| `gateway/trimmer.py`, `gateway/codeview.py`, `gateway/textcut.py` | cache-safe trimming of tool results: code, command output, search |
|
|
227
|
+
| `gateway/history.py`, `gateway/session.py` | cache-event trimming of old history and tool definitions; per-conversation memory |
|
|
228
|
+
|
|
229
|
+
## Ownership
|
|
230
|
+
|
|
231
|
+
| Folder | Owner |
|
|
232
|
+
|---|---|
|
|
233
|
+
| `tokencut/ingestion/`, `eval/` (SWE-bench loader) | Member A — Masooma Ali |
|
|
234
|
+
| `tokencut/graph/`, `api/` | Member B — Huma Taj |
|
|
235
|
+
| `tokencut/trimming/`, `eval/` (metrics) | Member C — Fatima Shakir |
|
|
236
|
+
|
|
237
|
+
## Setup (Windows PowerShell)
|
|
238
|
+
|
|
239
|
+
```powershell
|
|
240
|
+
git clone <repo-url>
|
|
241
|
+
cd tokencut
|
|
242
|
+
python -m venv .venv
|
|
243
|
+
.venv\Scripts\Activate.ps1
|
|
244
|
+
pip install -r requirements.txt
|
|
245
|
+
pytest
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
All tests in `tests/test_setup.py` must pass before you start coding.
|
|
249
|
+
|
|
250
|
+
**For PDFs with an agent, also install Poppler** (`pdftoppm` on PATH). Claude Code's Read tool
|
|
251
|
+
needs it to read PDF page ranges; it is a program, not a Python package, so it is not in
|
|
252
|
+
`requirements.txt`. Windows steps: `docs/TEAMMATE_SETUP.md` → "Poppler for PDFs";
|
|
253
|
+
macOS `brew install poppler`; Linux `sudo apt-get install poppler-utils`.
|
|
254
|
+
The first run of the token counter downloads a small tokenizer file (needs internet once).
|
|
255
|
+
|
|
256
|
+
## Workflow
|
|
257
|
+
|
|
258
|
+
1. `git checkout main && git pull`
|
|
259
|
+
2. `git checkout -b feat/<short-name>`
|
|
260
|
+
3. Code + tests, run `pytest` and `ruff check .`
|
|
261
|
+
4. Commit small, clear messages; push; open a Pull Request
|
|
262
|
+
5. One teammate reviews and approves, then merge
|
|
263
|
+
|
|
264
|
+
## Test fixture
|
|
265
|
+
|
|
266
|
+
`fixtures/sample_repo/` is a tiny fake project with deliberate traps (syntax error,
|
|
267
|
+
circular import, name clash, binary file…). See its README.
|