tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,455 @@
1
+ """Repository index for coding agents: symbol map, hybrid code search, exact symbols.
2
+
3
+ Runs NEXT TO the agent (as a local MCP server, see mcp.py) on the developer's working tree,
4
+ so uncommitted edits are included and no repository leaves the machine. The gateway still
5
+ only trims what is inside requests (team decision); this is a separate, optional component.
6
+
7
+ repo_map(prefix) directory layout + class / function signatures of the most-used files
8
+ find_code(query) chunks (whole functions / classes) ranked by graph distance from
9
+ what the query names + BM25 + file BM25 (+ embeddings if computed)
10
+ symbol(name) exact source of a symbol with line numbers, its callers and callees
11
+
12
+ Incremental: every file is remembered with its size, mtime and sha256; a refresh re-parses
13
+ only files whose content changed. BM25 is updated per file; the code graph (whole repo,
14
+ ~5 s on django) is rebuilt in a background thread and the last one is used meanwhile.
15
+ """
16
+
17
+ import hashlib
18
+ import json
19
+ import logging
20
+ import os
21
+ import threading
22
+ from collections.abc import Callable
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path, PurePosixPath
25
+
26
+ from tokencut.contracts import Chunk, ChunkKind, EdgeKind
27
+ from tokencut.graph.builder import build_graph
28
+ from tokencut.graph.queries import GraphIndex, file_of
29
+ from tokencut.ingestion.python_parser import PythonParser
30
+ from tokencut.ingestion.walker import list_python_files
31
+ from tokencut.repoindex.bm25 import Bm25Index, normalised
32
+ from tokencut.tokens import count_tokens
33
+ from tokencut.trimming.query import SYMBOL_WEIGHT, analyse_query
34
+ from tokencut.trimming.scoring import structural_scores, tokenize
35
+
36
+ log = logging.getLogger(__name__)
37
+
38
+ INDEX_VERSION = 1
39
+ DEFAULT_CACHE_DIR = Path(
40
+ os.environ.get("TOKENCUT_INDEX_DIR", Path.home() / ".cache" / "tokencut" / "repoindex"))
41
+ # Score = weighted sum of signals in [0, 1], times TEST_FACTOR for chunks in test files.
42
+ # (tunable) Chosen on the 40 dev tasks only (eval/repoindex_eval.py; DECISIONS change log).
43
+ W_STRUCTURAL = 0.2
44
+ W_TEXT = 0.4
45
+ W_FILE = 0.0
46
+ TEST_FACTOR = 0.7 # bug fixes rarely edit tests; they still show up when clearly relevant
47
+ W_EMBED = 0.3 # (not tuned) only when embeddings were computed for the repository
48
+ PSEUDO_MENTIONS = 3 # query names nothing: its best text matches act as the mentions
49
+ MAP_DIR_LINES = 40
50
+ MAP_METHODS_SHOWN = 12
51
+ MAP_LINES_PER_FILE = 12 # classes / functions listed per file (more files fit the budget)
52
+ MAX_SYMBOL_MATCHES = 4
53
+ MAX_RELATED = 12
54
+
55
+
56
+ @dataclass
57
+ class FileEntry:
58
+ size: int
59
+ mtime_ns: int
60
+ sha256: str
61
+ chunks: list[Chunk] = field(default_factory=list)
62
+
63
+
64
+ @dataclass
65
+ class Hit:
66
+ chunk: Chunk
67
+ score: float
68
+ structural: float = 0.0
69
+ text: float = 0.0
70
+ file: float = 0.0
71
+ embed: float = 0.0
72
+
73
+
74
+ def breadcrumb(chunk: Chunk) -> str:
75
+ """'django/db/models/query.py › QuerySet › only'"""
76
+ if chunk.kind == ChunkKind.MODULE:
77
+ return f"{chunk.file_path} › (module level)"
78
+ return " › ".join([chunk.file_path, *chunk.qualified_name.split(".")])
79
+
80
+
81
+ def numbered_code(chunk: Chunk) -> str:
82
+ """The chunk's source with line numbers, in the agent's Read format (' 12→code')."""
83
+ return "\n".join(f"{n:>6}→{line}"
84
+ for n, line in enumerate(chunk.code.split("\n"), chunk.start_line))
85
+
86
+
87
+ def _signature_line(chunk: Chunk) -> str:
88
+ sig = " ".join(part.strip() for part in chunk.signature.split("\n"))
89
+ return sig.rstrip(":").strip() or chunk.qualified_name.split(".")[-1]
90
+
91
+
92
+ def _cache_file(root: Path, cache_dir: Path) -> Path:
93
+ key = hashlib.sha1(str(root.resolve()).encode()).hexdigest()[:16]
94
+ return cache_dir / f"{root.resolve().name}-{key}.json"
95
+
96
+
97
+ class RepoIndex:
98
+ def __init__(self, root: Path | None = None, cache_file: Path | None = None):
99
+ self.root = root
100
+ self.cache_file = cache_file
101
+ self.files: dict[str, FileEntry] = {}
102
+ self.text = Bm25Index() # chunk documents: path + qualified name + code
103
+ self.file_text = Bm25Index() # file documents: path + all its code
104
+ self.by_id: dict[str, Chunk] = {}
105
+ self.embedder = None # optional: object with similarities(query, chunks) -> {id: cos}
106
+ self._version = 0
107
+ self._cache_dirty = False
108
+ self._graph: GraphIndex | None = None
109
+ self._callers: dict[str, list[str]] = {}
110
+ self._graph_version = -1
111
+ self._rebuild: threading.Thread | None = None
112
+ self._lock = threading.Lock()
113
+
114
+ # ---------- building ----------
115
+
116
+ @classmethod
117
+ def from_chunks(cls, chunks: list[Chunk]) -> "RepoIndex":
118
+ """An index over already-parsed chunks (offline evaluation; no files, no refresh)."""
119
+ index = cls()
120
+ by_file: dict[str, list[Chunk]] = {}
121
+ for c in chunks:
122
+ by_file.setdefault(c.file_path, []).append(c)
123
+ for path in sorted(by_file):
124
+ index._set_file(path, FileEntry(0, 0, by_file[path][0].file_hash, by_file[path]))
125
+ return index
126
+
127
+ @classmethod
128
+ def open(cls, root: str | Path, cache_dir: Path = DEFAULT_CACHE_DIR) -> "RepoIndex":
129
+ """Load the cached index of this working tree (if any) and bring it up to date."""
130
+ root = Path(root)
131
+ if not root.is_dir():
132
+ raise FileNotFoundError(f"repository folder not found: {root}")
133
+ index = cls(root, _cache_file(root, cache_dir))
134
+ index._load()
135
+ index.refresh()
136
+ return index
137
+
138
+ def _load(self) -> None:
139
+ if self.cache_file is None or not self.cache_file.is_file():
140
+ return
141
+ try:
142
+ data = json.loads(self.cache_file.read_text(encoding="utf-8"))
143
+ if data.get("version") != INDEX_VERSION:
144
+ return
145
+ for path, entry in sorted(data["files"].items()):
146
+ chunks = [Chunk.model_validate(c) for c in entry["chunks"]]
147
+ self._set_file(path, FileEntry(entry["size"], entry["mtime_ns"],
148
+ entry["sha256"], chunks))
149
+ except (OSError, ValueError, KeyError) as exc:
150
+ log.warning("index cache unreadable, rebuilding: %s", exc)
151
+ self.files.clear()
152
+ self.text, self.file_text, self.by_id = Bm25Index(), Bm25Index(), {}
153
+
154
+ def save(self) -> None:
155
+ if self.cache_file is None or not self._cache_dirty:
156
+ return
157
+ self.cache_file.parent.mkdir(parents=True, exist_ok=True)
158
+ data = {"version": INDEX_VERSION, "files": {
159
+ path: {"size": e.size, "mtime_ns": e.mtime_ns, "sha256": e.sha256,
160
+ "chunks": [c.model_dump(mode="json") for c in e.chunks]}
161
+ for path, e in sorted(self.files.items())}}
162
+ tmp = self.cache_file.with_suffix(".tmp")
163
+ tmp.write_text(json.dumps(data), encoding="utf-8")
164
+ tmp.replace(self.cache_file)
165
+ self._cache_dirty = False
166
+
167
+ def refresh(self) -> list[str]:
168
+ """Re-parse files whose content changed in the working tree. Returns changed paths."""
169
+ if self.root is None:
170
+ return []
171
+ current = list_python_files(self.root)
172
+ changed = []
173
+ parser = None
174
+ for rel in current:
175
+ try:
176
+ st = (self.root / rel).stat()
177
+ entry = self.files.get(rel)
178
+ if entry and entry.size == st.st_size and entry.mtime_ns == st.st_mtime_ns:
179
+ continue
180
+ source = (self.root / rel).read_bytes()
181
+ except OSError as exc:
182
+ log.warning("cannot read %s: %s", rel, exc)
183
+ continue
184
+ sha = hashlib.sha256(source).hexdigest()
185
+ self._cache_dirty = True
186
+ if entry and entry.sha256 == sha: # touched, not changed
187
+ entry.size, entry.mtime_ns = st.st_size, st.st_mtime_ns
188
+ continue
189
+ parser = parser or PythonParser()
190
+ self._set_file(rel, FileEntry(st.st_size, st.st_mtime_ns, sha,
191
+ parser.parse_file(rel, source)))
192
+ changed.append(rel)
193
+ for rel in sorted(set(self.files) - set(current)):
194
+ self._drop_file(rel)
195
+ changed.append(rel)
196
+ if changed:
197
+ self._version += 1
198
+ self._cache_dirty = True
199
+ return sorted(changed)
200
+
201
+ def _drop_file(self, path: str) -> None:
202
+ entry = self.files.pop(path, None)
203
+ if entry is None:
204
+ return
205
+ for c in entry.chunks:
206
+ self.text.remove(c.id)
207
+ self.by_id.pop(c.id, None)
208
+ self.file_text.remove(path)
209
+
210
+ def _set_file(self, path: str, entry: FileEntry) -> None:
211
+ self._drop_file(path)
212
+ self.files[path] = entry
213
+ words = tokenize(path)
214
+ for c in entry.chunks:
215
+ self.by_id[c.id] = c
216
+ self.text.add(c.id, tokenize(f"{c.file_path} {c.qualified_name} {c.code}"))
217
+ words.extend(tokenize(c.code))
218
+ self.file_text.add(path, words)
219
+ self._version += 1
220
+
221
+ def chunks(self) -> list[Chunk]:
222
+ return [c for path in sorted(self.files) for c in self.files[path].chunks]
223
+
224
+ # ---------- code graph ----------
225
+
226
+ def _build_graph(self, version: int) -> None:
227
+ graph = build_graph(self.chunks())
228
+ callers: dict[str, list[str]] = {}
229
+ for edge in graph.edges:
230
+ if edge.kind == EdgeKind.CALLS:
231
+ callers.setdefault(edge.target, []).append(edge.source)
232
+ with self._lock:
233
+ self._graph, self._callers, self._graph_version = GraphIndex(graph), callers, version
234
+
235
+ def graph(self, wait: bool = False) -> GraphIndex | None:
236
+ """The code graph; after a change the previous graph is returned while a new one is
237
+ built in the background (wait=True builds it now)."""
238
+ version = self._version
239
+ if self._graph_version == version:
240
+ return self._graph
241
+ if self._graph is None or wait:
242
+ self._build_graph(version)
243
+ return self._graph
244
+ current = self._graph
245
+ if self._rebuild is None or not self._rebuild.is_alive():
246
+ self._rebuild = threading.Thread(target=self._build_graph, args=(version,),
247
+ daemon=True)
248
+ self._rebuild.start()
249
+ return current
250
+
251
+ # ---------- search ----------
252
+
253
+ def search(self, query: str, k: int = 10, weights: tuple[float, ...] | None = None,
254
+ chunks: list[Chunk] | None = None) -> list[Hit]:
255
+ """Best k chunks for the query. `weights`: (structural, text, file, embed,
256
+ test factor). `chunks`: precomputed self.chunks() (speed)."""
257
+ w_struct, w_text, w_file, w_embed, test_factor = weights or (
258
+ W_STRUCTURAL, W_TEXT, W_FILE, W_EMBED, TEST_FACTOR)
259
+ chunks = chunks if chunks is not None else self.chunks()
260
+ words = tokenize(query)
261
+ text = normalised(self.text.scores(words))
262
+ file_text = normalised(self.file_text.scores(words))
263
+ mentions = analyse_query(query, chunks).weights
264
+ if not mentions:
265
+ best = sorted(text, key=lambda cid: (-text[cid], cid))[:PSEUDO_MENTIONS]
266
+ mentions = {cid: SYMBOL_WEIGHT for cid in best}
267
+ graph = self.graph()
268
+ if graph is not None:
269
+ structural = structural_scores(graph, mentions)
270
+ else:
271
+ structural = dict(mentions)
272
+ top = max(structural.values(), default=0.0)
273
+ structural = {cid: s / top for cid, s in structural.items()} if top > 0 else {}
274
+ embed: dict[str, float] = {}
275
+ if self.embedder is not None and w_embed > 0:
276
+ sims = self.embedder.similarities(query, chunks)
277
+ if sims:
278
+ lo, hi = min(sims.values()), max(sims.values())
279
+ embed = {cid: (s - lo) / (hi - lo) for cid, s in sims.items()} if hi > lo else {}
280
+ candidates = set(text) | set(structural) | {c.id for c in chunks
281
+ if c.file_path in file_text}
282
+ candidates |= set(sorted(embed, key=lambda cid: -embed[cid])[:500])
283
+ hits = []
284
+ for cid in candidates:
285
+ chunk = self.by_id.get(cid)
286
+ if chunk is None:
287
+ continue
288
+ s, t = structural.get(cid, 0.0), text.get(cid, 0.0)
289
+ f, e = file_text.get(chunk.file_path, 0.0), embed.get(cid, 0.0)
290
+ score = w_struct * s + w_text * t + w_file * f + (w_embed * e if embed else 0.0)
291
+ if is_test_file(chunk.file_path):
292
+ score *= test_factor
293
+ hits.append(Hit(chunk, score, s, t, f, e))
294
+ hits.sort(key=lambda h: (-h.score, h.chunk.id))
295
+ return hits[:k]
296
+
297
+ # ---------- what the agent sees ----------
298
+
299
+ def find_code(self, query: str, k: int = 8, max_tokens: int = 3000) -> str:
300
+ hits = self.search(query, k)
301
+ if not hits:
302
+ return "No matching code found."
303
+ out = [(f"{len(hits)} best matches (best first). Code is shown while it fits "
304
+ f"{max_tokens} tokens; use `symbol` for the rest.")]
305
+ used = count_tokens(out[0])
306
+ for i, hit in enumerate(hits, 1):
307
+ c = hit.chunk
308
+ head = f"\n{i}. {breadcrumb(c)} (lines {c.start_line}-{c.end_line})"
309
+ if c.kind != ChunkKind.MODULE and c.signature:
310
+ head += f"\n {_signature_line(c)}"
311
+ code = numbered_code(c)
312
+ cost = count_tokens(head) + count_tokens(code)
313
+ if used + cost <= max_tokens:
314
+ out += [head, code]
315
+ used += cost
316
+ else:
317
+ out.append(head)
318
+ used += count_tokens(head)
319
+ return "\n".join(out)
320
+
321
+ def _file_importance(self) -> dict[str, int]:
322
+ """Number of other non-test files whose code points at each file (calls / imports /
323
+ bases). Tests are left out: they would make test helpers look most important."""
324
+ graph = self.graph()
325
+ if graph is None:
326
+ return {}
327
+ users: dict[str, set[str]] = {}
328
+ for source, targets in graph.undirected.items():
329
+ src_file = file_of(source)
330
+ if is_test_file(src_file):
331
+ continue
332
+ for target in targets:
333
+ tgt_file = file_of(target)
334
+ if tgt_file != src_file:
335
+ users.setdefault(tgt_file, set()).add(src_file)
336
+ return {path: len(files) for path, files in users.items()}
337
+
338
+ def repo_map(self, prefix: str = "", max_tokens: int = 2000) -> str:
339
+ """Directory layout, then class / function signatures of the most-used files."""
340
+ prefix = prefix.strip("/")
341
+ paths = [p for p in sorted(self.files)
342
+ if not prefix or p == prefix or p.startswith(prefix + "/")]
343
+ if not paths:
344
+ return f"No Python files under '{prefix}'."
345
+ out = [f"# {len(paths)} Python files" + (f" under {prefix}/" if prefix else ""),
346
+ "## Directories (Python files below each)"]
347
+ out += _directory_lines(paths)
348
+ out.append("## Most-used files: classes and functions")
349
+ importance = self._file_importance()
350
+ ranked = sorted(paths, key=lambda p: (is_test_file(p), -importance.get(p, 0), p))
351
+ used = count_tokens("\n".join(out))
352
+ shown = 0
353
+ for path in ranked:
354
+ block = self._file_outline(path, importance.get(path, 0))
355
+ cost = count_tokens(block)
356
+ if used + cost > max_tokens:
357
+ continue
358
+ out.append(block)
359
+ used += cost
360
+ shown += 1
361
+ if shown < len(paths):
362
+ out.append(f"... {len(paths) - shown} more files not shown (pass a prefix, "
363
+ "or use find_code)")
364
+ return "\n".join(out)
365
+
366
+ def _file_outline(self, path: str, users: int) -> str:
367
+ chunks = self.files[path].chunks
368
+ lines = [f"{path}" + (f" (used by {users} files)" if users else "")]
369
+ methods: dict[str, list[str]] = {}
370
+ for c in chunks:
371
+ if c.kind == ChunkKind.METHOD and c.parent_id:
372
+ methods.setdefault(c.parent_id, []).append(c.qualified_name.split(".")[-1])
373
+ for c in chunks:
374
+ if c.kind == ChunkKind.CLASS and "." not in c.qualified_name:
375
+ names = methods.get(c.id, [])
376
+ listed = ", ".join(names[:MAP_METHODS_SHOWN])
377
+ more = f", +{len(names) - MAP_METHODS_SHOWN}" if len(names) > MAP_METHODS_SHOWN else ""
378
+ lines.append(f" {_signature_line(c)}" + (f": {listed}{more}" if names else ""))
379
+ elif c.kind == ChunkKind.FUNCTION:
380
+ lines.append(f" {_signature_line(c)}")
381
+ if len(lines) > MAP_LINES_PER_FILE + 1:
382
+ hidden = len(lines) - 1 - MAP_LINES_PER_FILE
383
+ lines = lines[: MAP_LINES_PER_FILE + 1] + [f" ... +{hidden} more (use symbol / find_code)"]
384
+ return "\n".join(lines)
385
+
386
+ def _match(self, name: str) -> list[Chunk]:
387
+ name = name.strip().replace("::", ".")
388
+ exact = [c for c in self.by_id.values()
389
+ if c.kind != ChunkKind.MODULE and (c.qualified_name == name or c.id == name
390
+ or c.qualified_name.endswith("." + name))]
391
+ if not exact:
392
+ last = name.split(".")[-1].lower()
393
+ exact = [c for c in self.by_id.values() if c.kind != ChunkKind.MODULE
394
+ and c.qualified_name.split(".")[-1].lower() == last]
395
+ return sorted(exact, key=lambda c: (c.file_path, c.start_line, c.id))
396
+
397
+ def symbol(self, name: str) -> str:
398
+ matches = self._match(name)
399
+ if not matches:
400
+ return f"No symbol named '{name}'. Try find_code."
401
+ if len(matches) > MAX_SYMBOL_MATCHES:
402
+ listed = "\n".join(f"- {breadcrumb(c)} (lines {c.start_line}-{c.end_line})"
403
+ for c in matches[:40])
404
+ more = f"\n... {len(matches) - 40} more" if len(matches) > 40 else ""
405
+ return (f"{len(matches)} symbols match '{name}'; ask again with a qualified name "
406
+ f"(e.g. Class.method):\n{listed}{more}")
407
+ graph = self.graph()
408
+ blocks = []
409
+ for c in matches:
410
+ block = [f"{breadcrumb(c)} (lines {c.start_line}-{c.end_line})", numbered_code(c)]
411
+ callers = sorted(set(self._callers.get(c.id, [])))
412
+ if callers:
413
+ block.append(_related("called by", callers))
414
+ if graph is not None and graph.depends_on.get(c.id):
415
+ block.append(_related("calls / inherits", sorted(set(graph.depends_on[c.id]))))
416
+ blocks.append("\n".join(block))
417
+ return "\n\n".join(blocks)
418
+
419
+
420
+ def is_test_file(path: str) -> bool:
421
+ parts = PurePosixPath(path).parts
422
+ name = parts[-1] if parts else ""
423
+ return (name.startswith("test_") or name.endswith("_test.py") or name == "conftest.py"
424
+ or any(part in ("tests", "test", "testing") for part in parts[:-1]))
425
+
426
+
427
+ def _directory_lines(paths: list[str]) -> list[str]:
428
+ """Directories with the number of Python files below them, at the deepest level that
429
+ fits MAP_DIR_LINES lines (deep trees such as 100 locale folders collapse into one)."""
430
+ lines: list[str] = []
431
+ for depth in range(4, 0, -1):
432
+ counts: dict[str, int] = {}
433
+ for p in paths:
434
+ parts = PurePosixPath(p).parts[:-1]
435
+ key = "/".join(parts[:depth]) if parts else "."
436
+ counts[key] = counts.get(key, 0) + 1
437
+ lines = [f"{d}/ ({counts[d]})" for d in sorted(counts)]
438
+ if len(lines) <= MAP_DIR_LINES:
439
+ return lines
440
+ return lines[:MAP_DIR_LINES] + [f"... {len(lines) - MAP_DIR_LINES} more (pass a prefix)"]
441
+
442
+
443
+ def _related(label: str, ids: list[str]) -> str:
444
+ shown = ", ".join(i.replace("::", ":") for i in ids[:MAX_RELATED])
445
+ more = f" (+{len(ids) - MAX_RELATED} more)" if len(ids) > MAX_RELATED else ""
446
+ return f"{label}: {shown}{more}"
447
+
448
+
449
+ def timed(fn: Callable[[], str]) -> str:
450
+ """Run a tool function; any failure becomes a short message instead of a crash."""
451
+ try:
452
+ return fn()
453
+ except Exception as exc: # the agent gets an answer, the server keeps running
454
+ log.exception("index tool failed")
455
+ return f"TokenCut index error: {type(exc).__name__}: {exc}"
@@ -0,0 +1,145 @@
1
+ """The repo index as a local MCP server (Model Context Protocol, stdio transport).
2
+
3
+ claude mcp add tokencut -- python -m tokencut.repoindex serve --repo .
4
+
5
+ The agent then has three tools (Claude Code names them mcp__tokencut__<name>). Every call
6
+ first refreshes the index from the working tree, so answers include uncommitted edits.
7
+ Messages are JSON-RPC 2.0, one per line on stdin / stdout; logs go to stderr.
8
+ """
9
+
10
+ import json
11
+ import logging
12
+ import sys
13
+ import time
14
+ from typing import IO
15
+
16
+ from tokencut.repoindex.index import RepoIndex, timed
17
+
18
+ log = logging.getLogger(__name__)
19
+
20
+ SERVER_INFO = {"name": "tokencut-repoindex", "version": "0.1.0"}
21
+ DEFAULT_PROTOCOL = "2025-06-18"
22
+ SAVE_EVERY_SECONDS = 60
23
+
24
+ TOOLS = [
25
+ {
26
+ "name": "repo_map",
27
+ "description": (
28
+ "Overview of the repository: directories, then the classes and function "
29
+ "signatures of its most-used files. Call it first in an unfamiliar repository, "
30
+ "or with a directory prefix to zoom in. Cheaper than listing and reading files."
31
+ ),
32
+ "inputSchema": {
33
+ "type": "object",
34
+ "properties": {
35
+ "prefix": {"type": "string", "description": "Only files under this directory"},
36
+ "max_tokens": {"type": "integer", "description": "Size limit (default 2000)"},
37
+ },
38
+ },
39
+ },
40
+ {
41
+ "name": "find_code",
42
+ "description": (
43
+ "Find the functions / classes relevant to a question, bug report, error message "
44
+ "or traceback. Returns the best matches with file, line range and code (with line "
45
+ "numbers). Use it instead of several grep + read rounds."
46
+ ),
47
+ "inputSchema": {
48
+ "type": "object",
49
+ "properties": {
50
+ "query": {"type": "string", "description": "Issue text, error, or symbol names"},
51
+ "k": {"type": "integer", "description": "Number of matches (default 8)"},
52
+ "max_tokens": {"type": "integer", "description": "Code shown (default 3000)"},
53
+ },
54
+ "required": ["query"],
55
+ },
56
+ },
57
+ {
58
+ "name": "symbol",
59
+ "description": (
60
+ "Exact current source of a function, method or class by name (e.g. "
61
+ "'QuerySet.only'), with line numbers, plus what calls it and what it calls."
62
+ ),
63
+ "inputSchema": {
64
+ "type": "object",
65
+ "properties": {"name": {"type": "string", "description": "Name or Class.method"}},
66
+ "required": ["name"],
67
+ },
68
+ },
69
+ ]
70
+
71
+
72
+ def _int(args: dict, key: str, default: int, lo: int, hi: int) -> int:
73
+ try:
74
+ return max(lo, min(hi, int(args.get(key, default))))
75
+ except (TypeError, ValueError):
76
+ return default
77
+
78
+
79
+ def call_tool(index: RepoIndex, name: str, args: dict) -> tuple[str, bool]:
80
+ """(text, is_error) for one tool call."""
81
+ index.refresh()
82
+ if name == "repo_map":
83
+ return timed(lambda: index.repo_map(str(args.get("prefix", "")),
84
+ _int(args, "max_tokens", 2000, 200, 20_000))), False
85
+ if name == "find_code":
86
+ query = str(args.get("query", "")).strip()
87
+ if not query:
88
+ return "find_code needs a query.", True
89
+ return timed(lambda: index.find_code(query, _int(args, "k", 8, 1, 50),
90
+ _int(args, "max_tokens", 3000, 200, 20_000))), False
91
+ if name == "symbol":
92
+ symbol = str(args.get("name", "")).strip()
93
+ if not symbol:
94
+ return "symbol needs a name.", True
95
+ return timed(lambda: index.symbol(symbol)), False
96
+ return f"Unknown tool {name!r}.", True
97
+
98
+
99
+ def handle(index: RepoIndex, message: dict) -> dict | None:
100
+ """Answer one JSON-RPC message (None for notifications)."""
101
+ method = message.get("method")
102
+ msg_id = message.get("id")
103
+ params = message.get("params") or {}
104
+ if msg_id is None:
105
+ return None # notification (e.g. notifications/initialized)
106
+ if method == "initialize":
107
+ result = {
108
+ "protocolVersion": params.get("protocolVersion") or DEFAULT_PROTOCOL,
109
+ "capabilities": {"tools": {"listChanged": False}},
110
+ "serverInfo": SERVER_INFO,
111
+ }
112
+ elif method == "ping":
113
+ result = {}
114
+ elif method == "tools/list":
115
+ result = {"tools": TOOLS}
116
+ elif method == "tools/call":
117
+ text, is_error = call_tool(index, str(params.get("name", "")),
118
+ params.get("arguments") or {})
119
+ result = {"content": [{"type": "text", "text": text}], "isError": is_error}
120
+ else:
121
+ return {"jsonrpc": "2.0", "id": msg_id,
122
+ "error": {"code": -32601, "message": f"Method not found: {method}"}}
123
+ return {"jsonrpc": "2.0", "id": msg_id, "result": result}
124
+
125
+
126
+ def serve(index: RepoIndex, stdin: IO[str] = sys.stdin, stdout: IO[str] = sys.stdout) -> None:
127
+ last_save = time.monotonic()
128
+ for line in stdin:
129
+ line = line.strip()
130
+ if not line:
131
+ continue
132
+ try:
133
+ message = json.loads(line)
134
+ except ValueError:
135
+ reply = {"jsonrpc": "2.0", "id": None,
136
+ "error": {"code": -32700, "message": "Parse error"}}
137
+ else:
138
+ reply = handle(index, message) if isinstance(message, dict) else None
139
+ if reply is not None:
140
+ stdout.write(json.dumps(reply) + "\n")
141
+ stdout.flush()
142
+ if time.monotonic() - last_save > SAVE_EVERY_SECONDS:
143
+ index.save()
144
+ last_save = time.monotonic()
145
+ index.save()
tokencut/tokens.py ADDED
@@ -0,0 +1,19 @@
1
+ """The ONE place where tokens are counted. Swap the tokenizer here only."""
2
+
3
+ from functools import lru_cache
4
+
5
+ import tiktoken
6
+
7
+ ENCODING_NAME = "o200k_base" # proxy tokenizer for all providers in Iteration 1
8
+
9
+
10
+ @lru_cache(maxsize=1)
11
+ def _encoding() -> tiktoken.Encoding:
12
+ # First call downloads the encoding file once, then it is cached on disk.
13
+ return tiktoken.get_encoding(ENCODING_NAME)
14
+
15
+
16
+ def count_tokens(text: str) -> int:
17
+ if not text:
18
+ return 0
19
+ return len(_encoding().encode(text, disallowed_special=()))
File without changes