tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,117 @@
1
+ """Relevance of every chunk to the query (IMPLEMENTATION_PLAN C2).
2
+
3
+ score = ALPHA * structural_score + (1 - ALPHA) * text_score
4
+ text_score BM25 over identifier sub-words, normalised to [0, 1]
5
+ structural_score weight / (1 + graph distance) from the chunks the query mentions
6
+ """
7
+
8
+ import re
9
+
10
+ from rank_bm25 import BM25Okapi
11
+
12
+ from tokencut.contracts import Chunk, CodeGraph, ScoredChunk
13
+ from tokencut.graph.queries import GraphIndex
14
+ from tokencut.trimming.query import SYMBOL_WEIGHT, QueryMentions, analyse_query
15
+
16
+ ALPHA = 0.5 # (tunable) log changes in the DECISIONS.md change log
17
+ PSEUDO_MENTIONS = 3 # fallback when the query names nothing: top-N chunks by text score
18
+
19
+ _IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
20
+ _CAMEL_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|[0-9]+")
21
+
22
+
23
+ def split_identifier(identifier: str) -> list[str]:
24
+ """'apply_discount' -> ['apply', 'discount']; 'HTTPServerError' -> ['http', 'server', 'error']."""
25
+ words = []
26
+ for part in identifier.split("_"):
27
+ for word in _CAMEL_RE.findall(part):
28
+ if len(word) >= 2:
29
+ words.append(word.lower())
30
+ return words
31
+
32
+
33
+ def tokenize(text: str) -> list[str]:
34
+ words = []
35
+ for identifier in _IDENT_RE.findall(text):
36
+ words.extend(split_identifier(identifier))
37
+ return words
38
+
39
+
40
+ def _bm25_normalised(documents: list[list[str]], query: str) -> list[float]:
41
+ """BM25 score of each document, divided by the max (all zeros if nothing matches)."""
42
+ query_words = tokenize(query)
43
+ if not query_words or not any(documents):
44
+ return [0.0] * len(documents)
45
+ raw = BM25Okapi(documents).get_scores(query_words)
46
+ clipped = [max(0.0, float(s)) for s in raw]
47
+ best = max(clipped, default=0.0)
48
+ if best <= 0:
49
+ return [0.0] * len(documents)
50
+ return [s / best for s in clipped]
51
+
52
+
53
+ def text_scores(chunks: list[Chunk], query: str) -> list[float]:
54
+ """BM25 score of each chunk, normalised to [0, 1]."""
55
+ documents = [tokenize(f"{c.file_path} {c.qualified_name} {c.code}") for c in chunks]
56
+ return _bm25_normalised(documents, query)
57
+
58
+
59
+ def file_text_scores(chunks: list[Chunk], query: str) -> dict[str, float]:
60
+ """BM25 score of each whole file (path + all its code), normalised to [0, 1].
61
+
62
+ Used to rank files for the evaluation pool: a file whose words match the issue as a
63
+ whole can matter even when no single chunk scores high.
64
+ """
65
+ files = sorted({c.file_path for c in chunks})
66
+ words: dict[str, list[str]] = {f: tokenize(f) for f in files}
67
+ for c in chunks:
68
+ words[c.file_path].extend(tokenize(c.code))
69
+ return dict(zip(files, _bm25_normalised([words[f] for f in files], query)))
70
+
71
+
72
+ def structural_scores(index: GraphIndex, weights: dict[str, float]) -> dict[str, float]:
73
+ """For each reachable chunk: max over mentioned chunks of weight / (1 + distance)."""
74
+ result: dict[str, float] = {}
75
+ for weight in sorted(set(weights.values()), reverse=True):
76
+ sources = sorted(cid for cid, w in weights.items() if w == weight)
77
+ for cid, distance in index.distances(sources).items():
78
+ result[cid] = max(result.get(cid, 0.0), weight / (1 + distance))
79
+ return result
80
+
81
+
82
+ def score_chunks(
83
+ chunks: list[Chunk],
84
+ graph: CodeGraph,
85
+ query: str,
86
+ mentions: QueryMentions | None = None,
87
+ index: GraphIndex | None = None,
88
+ ) -> list[ScoredChunk]:
89
+ """Score every chunk's relevance to the query (BM25 text score + graph distance)."""
90
+ if mentions is None:
91
+ mentions = analyse_query(query, chunks)
92
+ if index is None:
93
+ index = GraphIndex(graph)
94
+
95
+ texts = text_scores(chunks, query)
96
+ text_by_id = {c.id: t for c, t in zip(chunks, texts)}
97
+
98
+ weights = dict(mentions.weights)
99
+ if not weights:
100
+ # Fallback: the query names nothing, so the best text matches act as the mentions.
101
+ ranked = sorted(
102
+ (cid for cid, t in text_by_id.items() if t > 0), key=lambda cid: (-text_by_id[cid], cid)
103
+ )
104
+ weights = {cid: SYMBOL_WEIGHT for cid in ranked[:PSEUDO_MENTIONS]}
105
+ structural = structural_scores(index, weights)
106
+
107
+ scored = []
108
+ for c in chunks:
109
+ s = structural.get(c.id, 0.0)
110
+ t = text_by_id[c.id]
111
+ scored.append(
112
+ ScoredChunk(
113
+ chunk_id=c.id, score=ALPHA * s + (1 - ALPHA) * t, structural_score=s, text_score=t
114
+ )
115
+ )
116
+ scored.sort(key=lambda sc: (-sc.score, sc.chunk_id)) # DECISIONS #8: ties by chunk id
117
+ return scored
@@ -0,0 +1,61 @@
1
+ """Member C — Context scoring and 4-tier variant generation.
2
+
3
+ Note: works on chunks + graph, NOT on a repo path, so the same code can later
4
+ trim chunks cut out of an agent's request inside the gateway.
5
+ """
6
+
7
+ from tokencut.contracts import AGGRESSIVE_MIN_TOKENS, Chunk, CodeGraph, ContextVariant, ScoredChunk
8
+ from tokencut.graph.queries import GraphIndex
9
+ from tokencut.tokens import count_tokens
10
+ from tokencut.trimming import scoring
11
+ from tokencut.trimming.packing import TierPacker, build_pool, tier_budgets
12
+ from tokencut.trimming.query import analyse_query
13
+ from tokencut.trimming.render import render_variant
14
+
15
+
16
+ def score_chunks(chunks: list[Chunk], graph: CodeGraph, query: str) -> list[ScoredChunk]:
17
+ """Score every chunk's relevance to the query (BM25 text score + graph distance)."""
18
+ return scoring.score_chunks(chunks, graph, query)
19
+
20
+
21
+ def generate_variants(
22
+ chunks: list[Chunk],
23
+ graph: CodeGraph,
24
+ query: str,
25
+ full_cap_tokens: int | None = None,
26
+ aggressive_min_tokens: int = AGGRESSIVE_MIN_TOKENS,
27
+ ) -> list[ContextVariant]:
28
+ """Return exactly 4 variants (aggressive, moderate, conservative, full).
29
+
30
+ Must:
31
+ - respect each tier's token budget
32
+ - have token counts that never decrease from aggressive -> full
33
+ - be deterministic (break score ties by chunk id)
34
+
35
+ full_cap_tokens: None = gateway mode (everything is "full"); a number = evaluation mode
36
+ (top-ranked files up to that many tokens, e.g. EVAL_FULL_CONTEXT_CAP_TOKENS).
37
+ """
38
+ chunks = sorted(chunks, key=lambda c: (c.file_path, c.start_line, c.id))
39
+ index = GraphIndex(graph)
40
+ mentions = analyse_query(query, chunks)
41
+ scores = scoring.score_chunks(chunks, graph, query, mentions=mentions, index=index)
42
+
43
+ file_text = scoring.file_text_scores(chunks, query) if full_cap_tokens is not None else None
44
+ pool = build_pool(chunks, scores, full_cap_tokens, file_text)
45
+ full_tokens = count_tokens(render_variant(pool, {c.id for c in pool}, set()))
46
+ budgets = tier_budgets(full_tokens, aggressive_min_tokens)
47
+
48
+ order = {c.id: (c.file_path, c.start_line, c.id) for c in pool}
49
+ variants = []
50
+ for packed in TierPacker(pool, scores, index).pack(budgets):
51
+ variants.append(
52
+ ContextVariant(
53
+ tier=packed.tier,
54
+ budget_tokens=packed.budget_tokens,
55
+ kept_chunk_ids=sorted(packed.kept, key=order.__getitem__),
56
+ skeleton_chunk_ids=sorted(packed.skeletons, key=order.__getitem__),
57
+ token_count=packed.token_count,
58
+ rendered_text=packed.rendered_text,
59
+ )
60
+ )
61
+ return variants
tokencut/userconfig.py ADDED
@@ -0,0 +1,107 @@
1
+ """The installed tool's settings: ~/.tokencut/config.json (no secrets) and two private keys.
2
+
3
+ `tokencut setup` writes the file; `tokencut` (tokencut.launch) turns it into the gateway's
4
+ TOKENCUT_* environment. A TOKENCUT_* variable the user already set always wins, so every
5
+ gateway setting in CLAUDE.md / README keeps working. TOKENCUT_HOME moves the folder (tests).
6
+ """
7
+
8
+ import copy
9
+ import json
10
+ import os
11
+ import secrets
12
+ from pathlib import Path
13
+
14
+ DEFAULTS: dict = {
15
+ "version": 1,
16
+ "gateway": {
17
+ "port": 8787,
18
+ "mode": "trim",
19
+ "tier": "moderate",
20
+ "system_note": True,
21
+ "prompt_brief": True,
22
+ },
23
+ "litellm": {"enabled": False, "port": 8788}, # model connector; Claude + GPT go through it
24
+ "chatgpt": {"enabled": False, "models": ["gpt-6.1-sol", "gpt-6-luna", "gpt-6-astra"]},
25
+ }
26
+ KEY_FILES = {"tokencut": "tokencut_key", "litellm": "litellm_master_key"}
27
+
28
+
29
+ def home() -> Path:
30
+ return Path(os.environ.get("TOKENCUT_HOME") or Path.home() / ".tokencut")
31
+
32
+
33
+ def config_path() -> Path:
34
+ return home() / "config.json"
35
+
36
+
37
+ def _merge(base: dict, extra: dict) -> dict:
38
+ out = copy.deepcopy(base)
39
+ for key, value in extra.items():
40
+ out[key] = _merge(out[key], value) if isinstance(value, dict) and isinstance(
41
+ out.get(key), dict) else value
42
+ return out
43
+
44
+
45
+ def load() -> dict:
46
+ """The saved settings on top of the defaults (defaults alone if there is no file)."""
47
+ path = config_path()
48
+ saved = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
49
+ return _merge(DEFAULTS, saved)
50
+
51
+
52
+ def save(config: dict) -> Path:
53
+ path = config_path()
54
+ path.parent.mkdir(parents=True, exist_ok=True)
55
+ tmp = path.with_suffix(".tmp")
56
+ tmp.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
57
+ tmp.replace(path)
58
+ return path
59
+
60
+
61
+ def key(name: str, create: bool = True) -> str | None:
62
+ """A private random key from ~/.tokencut (owner-only file); created on first use."""
63
+ path = home() / KEY_FILES[name]
64
+ if path.exists():
65
+ return path.read_text(encoding="utf-8").strip() or None
66
+ if not create:
67
+ return None
68
+ path.parent.mkdir(parents=True, exist_ok=True)
69
+ value = ("tc-" if name == "tokencut" else "sk-") + secrets.token_hex(32)
70
+ path.write_text(value, encoding="utf-8")
71
+ path.chmod(0o600)
72
+ return value
73
+
74
+
75
+ def gateway_env(config: dict, litellm_url: str | None, environ: dict[str, str]) -> dict[str, str]:
76
+ """Environment for the gateway process: config values, never overriding the user's own."""
77
+ gw = config["gateway"]
78
+ wanted = {
79
+ "TOKENCUT_PORT": str(gw["port"]),
80
+ "TOKENCUT_MODE": gw["mode"],
81
+ "TOKENCUT_TIER": gw["tier"],
82
+ "TOKENCUT_SYSTEM_NOTE": "1" if gw["system_note"] else "0",
83
+ "TOKENCUT_PROMPT_BRIEF": "1" if gw["prompt_brief"] else "0",
84
+ "TOKENCUT_LOG_DIR": str(home() / "logs"),
85
+ }
86
+ tokencut_key = key("tokencut", create=False)
87
+ if tokencut_key:
88
+ wanted["TOKENCUT_KEY"] = tokencut_key
89
+ if litellm_url:
90
+ wanted["TOKENCUT_GPT_UPSTREAM"] = litellm_url
91
+ wanted["TOKENCUT_CLAUDE_VIA_LITELLM"] = "1"
92
+ litellm_key = key("litellm", create=False)
93
+ if litellm_key:
94
+ wanted["TOKENCUT_GPT_KEY"] = litellm_key
95
+ return {**wanted, **environ}
96
+
97
+
98
+ def agent_env(config: dict, environ: dict[str, str]) -> dict[str, str]:
99
+ """Environment for Claude Code: talk to the gateway and send the TokenCut key."""
100
+ port = environ.get("TOKENCUT_PORT", str(config["gateway"]["port"]))
101
+ out = {**environ, "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}/anthropic"}
102
+ tokencut_key = key("tokencut", create=False)
103
+ if tokencut_key:
104
+ headers = [line for line in environ.get("ANTHROPIC_CUSTOM_HEADERS", "").splitlines()
105
+ if line.strip() and not line.lower().startswith("x-tokencut-key:")]
106
+ out["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join([*headers, f"x-tokencut-key: {tokencut_key}"])
107
+ return out
tokencut/wizard.py ADDED
@@ -0,0 +1,94 @@
1
+ """`tokencut setup`: first-run questions and one-time installs.
2
+
3
+ 1. Claude Code must be installed (the user installs it; we print the official command).
4
+ 2. Ask whether to add the ChatGPT plan (GPT models in Claude Code via `/model gpt-…`).
5
+ 3. Create the private keys, install the hidden model connector (LiteLLM), sign in to ChatGPT.
6
+ 4. Download the prompt router model once (task briefs), so the first start is fast.
7
+ 5. Save ~/.tokencut/config.json.
8
+ Claude itself needs no step here: Claude Code asks for its own login on first start, and that
9
+ login travels through TokenCut unchanged. Every failure leaves a working Claude-only setup.
10
+ """
11
+
12
+ import os
13
+ import shutil
14
+ from collections.abc import Callable
15
+
16
+ from tokencut import litellm_proxy, userconfig
17
+
18
+ CLAUDE_INSTALL = {
19
+ "posix": "curl -fsSL https://claude.ai/install.sh | bash",
20
+ "nt": "irm https://claude.ai/install.ps1 | iex",
21
+ }
22
+
23
+
24
+ def _yes(answer: str, default: bool) -> bool:
25
+ answer = answer.strip().lower()
26
+ return default if not answer else answer in ("y", "yes")
27
+
28
+
29
+ def _download_router(out: Callable[[str], None]) -> bool:
30
+ from tokencut.gateway.task_router import load_router
31
+
32
+ out("Downloading the prompt model (about 135 MB, one time) ...")
33
+ return load_router() is not None
34
+
35
+
36
+ def run_setup(
37
+ chatgpt: bool | None = None,
38
+ ask: Callable[[str], str] = input,
39
+ out: Callable[[str], None] = print,
40
+ which: Callable[[str], str | None] = shutil.which,
41
+ os_name: str | None = None,
42
+ install_connector: Callable[[], bool] = litellm_proxy.install,
43
+ connector_ready: Callable[[], bool] = litellm_proxy.installed,
44
+ chatgpt_ready: Callable[[], bool] = litellm_proxy.chatgpt_logged_in,
45
+ chatgpt_login: Callable[[], bool] = litellm_proxy.chatgpt_login,
46
+ download_router: Callable[[Callable[[str], None]], bool] = _download_router,
47
+ ) -> int:
48
+ if which("claude") is None:
49
+ out("TokenCut works inside Claude Code, which is not installed yet. Install it with:")
50
+ out(f" {CLAUDE_INSTALL.get(os_name or os.name, CLAUDE_INSTALL['posix'])}")
51
+ out("then run `tokencut setup` again.")
52
+ return 1
53
+ config = userconfig.load()
54
+ if chatgpt is None:
55
+ chatgpt = _yes(ask("Also use your ChatGPT plan for GPT models? [y/N] "), False)
56
+
57
+ userconfig.key("tokencut")
58
+ userconfig.key("litellm")
59
+
60
+ ready = connector_ready()
61
+ if not ready:
62
+ out("Installing the model connector (one time, 1-3 minutes) ...")
63
+ ready = install_connector()
64
+ if not ready:
65
+ out("Could not install the model connector; Claude will connect directly, "
66
+ "GPT models are off. Run `tokencut setup` again to retry.")
67
+ config["litellm"]["enabled"] = ready
68
+
69
+ config["chatgpt"]["enabled"] = False
70
+ if chatgpt and ready:
71
+ if not chatgpt_ready():
72
+ out("ChatGPT sign-in: open the link below, sign in, and enter the code.")
73
+ if not chatgpt_login():
74
+ out("ChatGPT sign-in did not finish; GPT models are off for now.")
75
+ config["chatgpt"]["enabled"] = chatgpt_ready()
76
+ elif chatgpt:
77
+ out("GPT models need the model connector; they stay off for now.")
78
+
79
+ if config["gateway"]["prompt_brief"] and not download_router(out):
80
+ out("Prompt model unavailable; task briefs are off (everything else works).")
81
+ config["gateway"]["prompt_brief"] = False
82
+
83
+ if ready:
84
+ litellm_proxy.write_config(config)
85
+ path = userconfig.save(config)
86
+ models = "Claude" + (" + ChatGPT (" + ", ".join(config["chatgpt"]["models"]) + ")"
87
+ if config["chatgpt"]["enabled"] else "")
88
+ out(f"\nTokenCut is ready ({path}).")
89
+ out(f" Models: {models}")
90
+ out(f" Trimming: {config['gateway']['mode']} ({config['gateway']['tier']}), "
91
+ f"system note: {'on' if config['gateway']['system_note'] else 'off'}, "
92
+ f"task briefs: {'on' if config['gateway']['prompt_brief'] else 'off'}")
93
+ out("Run `tokencut` in your project folder to open Claude Code through TokenCut.")
94
+ return 0
@@ -0,0 +1,267 @@
1
+ Metadata-Version: 2.4
2
+ Name: tokencut-proxy
3
+ Version: 0.1.0
4
+ Summary: Cache-aware gateway that trims the context coding agents send to the model
5
+ Author: Masooma Ali, Huma Taj, Fatima Shakir
6
+ Project-URL: Repository, https://github.com/Masooma-Ali/TokenCut
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ Requires-Dist: tree-sitter<0.27,>=0.26
10
+ Requires-Dist: tree-sitter-python<0.26,>=0.25
11
+ Requires-Dist: networkx<4,>=3.6
12
+ Requires-Dist: pydantic<3,>=2.13
13
+ Requires-Dist: tiktoken<1,>=0.14
14
+ Requires-Dist: rank-bm25<0.3,>=0.2.2
15
+ Requires-Dist: fastapi<1,>=0.142
16
+ Requires-Dist: uvicorn<1,>=0.53
17
+ Requires-Dist: httpx2<3,>=2.13
18
+ Requires-Dist: pillow<14,>=13
19
+ Requires-Dist: pypdf<7,>=6.21
20
+ Requires-Dist: numpy<3,>=2.4
21
+ Requires-Dist: onnxruntime<2,>=1.30
22
+ Requires-Dist: tokenizers<1,>=0.23
23
+ Requires-Dist: huggingface-hub<2,>=1.33
24
+ Provides-Extra: embed
25
+ Requires-Dist: fastembed<1,>=0.9.1; extra == "embed"
26
+ Provides-Extra: dev
27
+ Requires-Dist: fastembed==0.9.1; extra == "dev"
28
+ Requires-Dist: unidiff==1.0.1; extra == "dev"
29
+ Requires-Dist: pytest==9.1.1; extra == "dev"
30
+ Requires-Dist: ruff==0.16.10; extra == "dev"
31
+
32
+ # TokenCut
33
+
34
+ Cost-aware joint context-and-model routing gateway for AI coding agents (FYP, FAST NUCES Islamabad).
35
+
36
+ **Iteration 1:** repository parsing, code-graph construction, and 4-tier context-variant generation.
37
+
38
+ ## Install and use (one command)
39
+ Needs [Claude Code](https://claude.ai/code) installed first. Then:
40
+ ```bash
41
+ curl -fsSL https://github.com/Masooma-Ali/TokenCut/releases/latest/download/install.sh | sh # macOS / Linux
42
+ ```
43
+ ```powershell
44
+ irm https://github.com/Masooma-Ali/TokenCut/releases/latest/download/install.ps1 | iex # Windows
45
+ ```
46
+ The installer adds [uv](https://docs.astral.sh/uv/) if missing, installs the `tokencut` command
47
+ (PyPI package `tokencut-proxy`) and runs `tokencut setup`, which asks whether to add your ChatGPT
48
+ plan, installs the hidden model connector (LiteLLM) and downloads the prompt model once.
49
+ Then, in any project folder:
50
+
51
+ | Command | What it does |
52
+ |---|---|
53
+ | `tokencut` | opens Claude Code through TokenCut (trimming, system note, task briefs, routing); `/model gpt-6.1-sol` for GPT if set up |
54
+ | `tokencut setup` | change logins / re-run one-time installs (`--chatgpt`, `--no-chatgpt`) |
55
+ | `tokencut status` | running? tokens removed, billed usage |
56
+ | `tokencut logs [gateway\|connector]` | last lines of a process log |
57
+
58
+ Settings: `~/.tokencut/config.json` (no secrets); every `TOKENCUT_*` variable below overrides it.
59
+ Claude keeps Claude Code's own login; the ChatGPT token stays in LiteLLM's file; a local
60
+ TokenCut key (`~/.tokencut/tokencut_key`) stops other programs from using the gateway.
61
+ Releases: push a tag `vX.Y.Z` (`.github/workflows/release.yml` tests, publishes to PyPI and
62
+ creates a GitHub Release with the install scripts attached; the `releases/latest/download/…`
63
+ links above always point to the newest one). The links work only while the repository is public.
64
+
65
+ ## Pipeline
66
+
67
+ ```
68
+ repo ──► ingestion.parse_repo() ──► list[Chunk]
69
+ │
70
+ ▼
71
+ graph.build_graph() ──► CodeGraph
72
+ │
73
+ query ───────────────►▼
74
+ trimming.generate_variants() ──► 4 × ContextVariant
75
+ ```
76
+
77
+ All shared data formats live in `tokencut/contracts.py`. Changing that file needs a PR approved by both other members.
78
+
79
+ ## Usage
80
+
81
+ ```python
82
+ from tokencut.ingestion.parser import parse_repo
83
+ from tokencut.graph.builder import build_graph
84
+ from tokencut.trimming.variants import generate_variants
85
+
86
+ chunks = parse_repo("fixtures/sample_repo")
87
+ graph = build_graph(chunks)
88
+ for v in generate_variants(chunks, graph, "Cart.total applies the discount wrong"):
89
+ print(v.tier.value, v.token_count, "/", v.budget_tokens)
90
+ ```
91
+
92
+ API: `uvicorn api.main:app --reload`, then `POST /context/variants` with
93
+ `{"repo_path": "...", "query": "...", "include_text": true}`.
94
+
95
+ Evaluation on SWE-bench Lite (clones repos into `swebench_repos/`, caches in `cache/`):
96
+
97
+ ```
98
+ python -m eval.run_eval --limit 3 # smoke run
99
+ python -m eval.run_eval # all tasks in eval/tasks_iter1.json
100
+ ```
101
+
102
+ Results go to `eval/results/iter1_results.csv` and `eval/results/iter1_summary.md`.
103
+
104
+ ## Benchmark: TokenCut vs baselines
105
+
106
+ `eval/benchmark.py` compares TokenCut with BM25, dense embeddings (bge-small), naive
107
+ truncation, random and SWE-bench file retrieval at **equal token budgets** (15 / 35 / 60 %
108
+ of a 32k context), in two settings: pure trimming of a shared neutral 32k context, and
109
+ selection from the whole repository. Reports gold-chunk recall with 95% bootstrap CIs and
110
+ paired sign tests. Tasks: 40 dev (used for tuning) and 81 held-out test tasks
111
+ (`eval/tasks_iter2.json`).
112
+
113
+ ```
114
+ python -m eval.prepare --set test # clone + parse once
115
+ python -m eval.benchmark --set test # no embeddings
116
+ python -m eval.benchmark --set test --methods tokencut,tokencut_hybrid,bm25,embed,random,head,bm25_files
117
+ ```
118
+
119
+ Results: `eval/results/benchmark_<set>.md` / `.csv`.
120
+
121
+ ## Gateway (Iteration 2)
122
+
123
+ The gateway sits between your coding agent and the provider. It streams the reply back and
124
+ records what each request contained (tokens per kind: system prompt, tools, chat, file
125
+ reads, command output…) plus the tokens the provider actually billed, including cached
126
+ tokens. In shadow mode (default) requests are forwarded unchanged and the stats show what
127
+ trimming *would* save; in trim mode the trimmed request is sent.
128
+
129
+ ```
130
+ python -m tokencut.gateway # listens on 127.0.0.1:8787
131
+ ```
132
+
133
+ Point an agent at it:
134
+
135
+ | Agent / SDK | Setting |
136
+ |---|---|
137
+ | Claude Code (Anthropic) | `ANTHROPIC_BASE_URL=http://127.0.0.1:8787/anthropic` |
138
+ | OpenAI SDKs, Cline, OpenRouter-style clients | base URL `http://127.0.0.1:8787/openai/v1` |
139
+ | Gemini (google-genai / Gemini CLI) | base URL `http://127.0.0.1:8787/gemini` |
140
+
141
+ Your API key travels in the request headers to the provider as usual; it is never logged.
142
+ See what was measured at `http://127.0.0.1:8787/tokencut/stats`; per-request numbers go to
143
+ `logs/gateway/requests-<date>.jsonl` (gitignored).
144
+
145
+ | Variable | Default | Meaning |
146
+ |---|---|---|
147
+ | `TOKENCUT_ANTHROPIC_UPSTREAM` / `_OPENAI_` / `_GEMINI_` | official API URLs | send to another compatible endpoint (e.g. OpenRouter) |
148
+ | `TOKENCUT_LOG_DIR` | `logs/gateway` | where numbers are logged (empty = off) |
149
+ | `TOKENCUT_KEY` | none | if set, proxy requests must send it in `x-tokencut-key` (the `tokencut` command does this; the header is never forwarded) |
150
+ | `TOKENCUT_CAPTURE` | off | `1` = also save full request bodies locally, for parser test fixtures (contain your prompts and code) |
151
+ | `TOKENCUT_HOST` / `TOKENCUT_PORT` | `127.0.0.1` / `8787` | listen address |
152
+ | `TOKENCUT_MODE` | `shadow` | `shadow` = forward unchanged, report possible savings; `trim` = send the trimmed request |
153
+ | `TOKENCUT_TIER` | `moderate` | `aggressive` / `moderate` / `conservative` |
154
+ | `TOKENCUT_MIN_TRIM_TOKENS` | `2000` | tool results smaller than this are never trimmed |
155
+ | `TOKENCUT_CACHE_EVENTS` | `1` | `0` = never re-trim history at cache events |
156
+ | `TOKENCUT_CACHE_TTL` | `300` | seconds of idle after which the provider cache counts as expired |
157
+ | `TOKENCUT_PRUNE_TOOLS` | off | `1` = at cache events, send only tools used so far + core tools |
158
+ | `TOKENCUT_CORE_TOOLS` | Claude Code / Gemini CLI basics | comma-separated tools always kept when pruning |
159
+ | `TOKENCUT_TASK_MASK` | **on** | when the user starts a new task, earlier tasks' large tool outputs become notes (only if the cache rewrite pays off); `0` = off |
160
+ | `TOKENCUT_TASK_MASK_TOKENS` | `300` | smallest tool output masked at a new task |
161
+ | `TOKENCUT_CCR` | off | `1` = every shortened output gets a reference; the gateway offers `tokencut_retrieve` and answers it with the exact original lines |
162
+ | `TOKENCUT_LAZY_TOOLS` | off | `1` = every request carries only core / used tools + a catalog tool `tokencut_enable_tools` for the rest |
163
+ | `TOKENCUT_SYSTEM_NOTE` | off | `1` = trim mode adds a fixed ~125-token note to the system prompt: how to treat `[TokenCut: …]` markers (re-read only the omitted lines, do not re-run commands for cut log lines, keep test output short) |
164
+ | `TOKENCUT_PROMPT_BRIEF` | off | `1` = trim mode classifies each typed user message (rules + the ONNX prompt router) and, when confident, appends a short task brief (`<tokencut_task_brief type="bugfix">…`, 48–71 tokens); agent requests only; never repeated for the same type in a row |
165
+ | `TOKENCUT_BRIEF_THRESHOLD` | `0.87` | minimum router probability for a brief (chosen on dev, RESULTS_REPORT §11) |
166
+ | `TOKENCUT_ROUTER_DIR` | — | local folder with `router.onnx`, `router.onnx.data`, `tokenizer.json`, `router_config.json` |
167
+ | `TOKENCUT_ROUTER_REPO` | `humataj/tokencut-router` | public Hugging Face repo the model is downloaded from (once, no login, to `~/.cache/tokencut/router`) when no local folder is set |
168
+
169
+ **Trimming (phase 3).** Only large tool results are shortened: Python file contents
170
+ (through the Iteration 1 engine; kept lines stay verbatim, gaps become
171
+ `[TokenCut: lines 40-95 of x.py omitted …]` markers so the model can re-read them),
172
+ command output (failures, errors and the summary are kept) and search results.
173
+ System prompt, tools, chat text and tool calls are never touched; files the user named or
174
+ the agent already edited stay whole. A result's trim depends only on earlier conversation,
175
+ so it is identical on every later turn and the provider's prompt cache keeps working.
176
+
177
+ **Cache-event trimming.** The cache is lost anyway when the model changes, after the cache
178
+ expires, or when the context is nearly full. At those moments TokenCut also shrinks old
179
+ history: old command/test/search output becomes a note with its error lines, older reads of
180
+ a file that was read again or edited become a short note, and (opt-in) unused tool
181
+ definitions are dropped. The last few turns are never touched. The decisions are remembered
182
+ per conversation and re-sent identically until the next event; a file the agent re-reads
183
+ after its old read was removed is protected from then on.
184
+
185
+ **Reversible trimming (P1, `TOKENCUT_CCR=1`).** Every shortened output ends with
186
+ `[TokenCut ref tc_…]`. The gateway adds a `tokencut_retrieve(ref, start_line, end_line)` tool;
187
+ the agent does not know it and answers "no such tool", and the gateway replaces that answer
188
+ with the exact original lines before forwarding. References are hashes of the original, so
189
+ they are identical every turn (cache-safe) and survive restarts.
190
+
191
+ **Lazy tool loading (P1, `TOKENCUT_LAZY_TOOLS=1`).** Each request carries only the core tools
192
+ (`TOKENCUT_CORE_TOOLS`), tools already used, and one catalog tool `tokencut_enable_tools`
193
+ listing the rest; enabling one adds it from that request on (one cache rewrite).
194
+
195
+ ## Repo index for the agent (P1, local MCP server)
196
+
197
+ Not part of the gateway: a local MCP server that indexes the working tree (uncommitted
198
+ edits included) and gives the agent `repo_map`, `find_code` and `symbol`, so it can find
199
+ code in one call instead of several grep + read rounds.
200
+
201
+ ```bash
202
+ python -m tokencut.repoindex build --repo . # first index (cached, incremental after)
203
+ claude mcp add tokencut -- python -m tokencut.repoindex serve --repo .
204
+ python -m tokencut.repoindex find --repo . "QuerySet.only() crashes with select_related"
205
+ ```
206
+ `--embed` adds dense embeddings (computed once in the background, cached per chunk).
207
+ Offline check on SWE-bench tasks: `python -m eval.repoindex_eval --set dev|test`.
208
+
209
+ ## Code map
210
+
211
+ | Module | What it does |
212
+ |---|---|
213
+ | `ingestion/walker.py` | file list (`git ls-files`, skip rules from DECISIONS #12) |
214
+ | `ingestion/python_parser.py` | tree-sitter chunker (module / class / function / method) |
215
+ | `graph/modules.py` | module names, import parsing, relative imports, `src/` layout |
216
+ | `graph/builder.py` | contains / imports / inherits / calls edges (DECISIONS #7) |
217
+ | `graph/queries.py` | `GraphIndex`: dependency closure, BFS distances |
218
+ | `trimming/query.py` | file / symbol / traceback mentions in the query |
219
+ | `trimming/scoring.py` | BM25 + graph-distance score |
220
+ | `trimming/packing.py` | full pool, tier budgets, nested greedy packing, skeletons |
221
+ | `trimming/render.py` | text sent to the model |
222
+ | `eval/loader.py`, `eval/metrics.py`, `eval/run_eval.py` | SWE-bench Lite subset, gold chunks, recall metrics |
223
+ | `gateway/app.py` | proxy server: forward, stream back, measure (shadow mode) |
224
+ | `gateway/adapters/` | Anthropic / OpenAI / Gemini request JSON → `Segment` list |
225
+ | `gateway/classify.py`, `gateway/usage.py`, `gateway/records.py` | tool-result kinds, billed usage from responses, request log |
226
+ | `gateway/trimmer.py`, `gateway/codeview.py`, `gateway/textcut.py` | cache-safe trimming of tool results: code, command output, search |
227
+ | `gateway/history.py`, `gateway/session.py` | cache-event trimming of old history and tool definitions; per-conversation memory |
228
+
229
+ ## Ownership
230
+
231
+ | Folder | Owner |
232
+ |---|---|
233
+ | `tokencut/ingestion/`, `eval/` (SWE-bench loader) | Member A — Masooma Ali |
234
+ | `tokencut/graph/`, `api/` | Member B — Huma Taj |
235
+ | `tokencut/trimming/`, `eval/` (metrics) | Member C — Fatima Shakir |
236
+
237
+ ## Setup (Windows PowerShell)
238
+
239
+ ```powershell
240
+ git clone <repo-url>
241
+ cd tokencut
242
+ python -m venv .venv
243
+ .venv\Scripts\Activate.ps1
244
+ pip install -r requirements.txt
245
+ pytest
246
+ ```
247
+
248
+ All tests in `tests/test_setup.py` must pass before you start coding.
249
+
250
+ **For PDFs with an agent, also install Poppler** (`pdftoppm` on PATH). Claude Code's Read tool
251
+ needs it to read PDF page ranges; it is a program, not a Python package, so it is not in
252
+ `requirements.txt`. Windows steps: `docs/TEAMMATE_SETUP.md` → "Poppler for PDFs";
253
+ macOS `brew install poppler`; Linux `sudo apt-get install poppler-utils`.
254
+ The first run of the token counter downloads a small tokenizer file (needs internet once).
255
+
256
+ ## Workflow
257
+
258
+ 1. `git checkout main && git pull`
259
+ 2. `git checkout -b feat/<short-name>`
260
+ 3. Code + tests, run `pytest` and `ruff check .`
261
+ 4. Commit small, clear messages; push; open a Pull Request
262
+ 5. One teammate reviews and approves, then merge
263
+
264
+ ## Test fixture
265
+
266
+ `fixtures/sample_repo/` is a tiny fake project with deliberate traps (syntax error,
267
+ circular import, name clash, binary file…). See its README.