tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
"""Repository index for coding agents: symbol map, hybrid code search, exact symbols.
|
|
2
|
+
|
|
3
|
+
Runs NEXT TO the agent (as a local MCP server, see mcp.py) on the developer's working tree,
|
|
4
|
+
so uncommitted edits are included and no repository leaves the machine. The gateway still
|
|
5
|
+
only trims what is inside requests (team decision); this is a separate, optional component.
|
|
6
|
+
|
|
7
|
+
repo_map(prefix) directory layout + class / function signatures of the most-used files
|
|
8
|
+
find_code(query) chunks (whole functions / classes) ranked by graph distance from
|
|
9
|
+
what the query names + BM25 + file BM25 (+ embeddings if computed)
|
|
10
|
+
symbol(name) exact source of a symbol with line numbers, its callers and callees
|
|
11
|
+
|
|
12
|
+
Incremental: every file is remembered with its size, mtime and sha256; a refresh re-parses
|
|
13
|
+
only files whose content changed. BM25 is updated per file; the code graph (whole repo,
|
|
14
|
+
~5 s on django) is rebuilt in a background thread and the last one is used meanwhile.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import json
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import threading
|
|
22
|
+
from collections.abc import Callable
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from pathlib import Path, PurePosixPath
|
|
25
|
+
|
|
26
|
+
from tokencut.contracts import Chunk, ChunkKind, EdgeKind
|
|
27
|
+
from tokencut.graph.builder import build_graph
|
|
28
|
+
from tokencut.graph.queries import GraphIndex, file_of
|
|
29
|
+
from tokencut.ingestion.python_parser import PythonParser
|
|
30
|
+
from tokencut.ingestion.walker import list_python_files
|
|
31
|
+
from tokencut.repoindex.bm25 import Bm25Index, normalised
|
|
32
|
+
from tokencut.tokens import count_tokens
|
|
33
|
+
from tokencut.trimming.query import SYMBOL_WEIGHT, analyse_query
|
|
34
|
+
from tokencut.trimming.scoring import structural_scores, tokenize
|
|
35
|
+
|
|
36
|
+
log = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
INDEX_VERSION = 1
|
|
39
|
+
DEFAULT_CACHE_DIR = Path(
|
|
40
|
+
os.environ.get("TOKENCUT_INDEX_DIR", Path.home() / ".cache" / "tokencut" / "repoindex"))
|
|
41
|
+
# Score = weighted sum of signals in [0, 1], times TEST_FACTOR for chunks in test files.
|
|
42
|
+
# (tunable) Chosen on the 40 dev tasks only (eval/repoindex_eval.py; DECISIONS change log).
|
|
43
|
+
W_STRUCTURAL = 0.2
|
|
44
|
+
W_TEXT = 0.4
|
|
45
|
+
W_FILE = 0.0
|
|
46
|
+
TEST_FACTOR = 0.7 # bug fixes rarely edit tests; they still show up when clearly relevant
|
|
47
|
+
W_EMBED = 0.3 # (not tuned) only when embeddings were computed for the repository
|
|
48
|
+
PSEUDO_MENTIONS = 3 # query names nothing: its best text matches act as the mentions
|
|
49
|
+
MAP_DIR_LINES = 40
|
|
50
|
+
MAP_METHODS_SHOWN = 12
|
|
51
|
+
MAP_LINES_PER_FILE = 12 # classes / functions listed per file (more files fit the budget)
|
|
52
|
+
MAX_SYMBOL_MATCHES = 4
|
|
53
|
+
MAX_RELATED = 12
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class FileEntry:
|
|
58
|
+
size: int
|
|
59
|
+
mtime_ns: int
|
|
60
|
+
sha256: str
|
|
61
|
+
chunks: list[Chunk] = field(default_factory=list)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass
|
|
65
|
+
class Hit:
|
|
66
|
+
chunk: Chunk
|
|
67
|
+
score: float
|
|
68
|
+
structural: float = 0.0
|
|
69
|
+
text: float = 0.0
|
|
70
|
+
file: float = 0.0
|
|
71
|
+
embed: float = 0.0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def breadcrumb(chunk: Chunk) -> str:
|
|
75
|
+
"""'django/db/models/query.py › QuerySet › only'"""
|
|
76
|
+
if chunk.kind == ChunkKind.MODULE:
|
|
77
|
+
return f"{chunk.file_path} › (module level)"
|
|
78
|
+
return " › ".join([chunk.file_path, *chunk.qualified_name.split(".")])
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def numbered_code(chunk: Chunk) -> str:
|
|
82
|
+
"""The chunk's source with line numbers, in the agent's Read format (' 12→code')."""
|
|
83
|
+
return "\n".join(f"{n:>6}→{line}"
|
|
84
|
+
for n, line in enumerate(chunk.code.split("\n"), chunk.start_line))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _signature_line(chunk: Chunk) -> str:
|
|
88
|
+
sig = " ".join(part.strip() for part in chunk.signature.split("\n"))
|
|
89
|
+
return sig.rstrip(":").strip() or chunk.qualified_name.split(".")[-1]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _cache_file(root: Path, cache_dir: Path) -> Path:
|
|
93
|
+
key = hashlib.sha1(str(root.resolve()).encode()).hexdigest()[:16]
|
|
94
|
+
return cache_dir / f"{root.resolve().name}-{key}.json"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class RepoIndex:
|
|
98
|
+
def __init__(self, root: Path | None = None, cache_file: Path | None = None):
|
|
99
|
+
self.root = root
|
|
100
|
+
self.cache_file = cache_file
|
|
101
|
+
self.files: dict[str, FileEntry] = {}
|
|
102
|
+
self.text = Bm25Index() # chunk documents: path + qualified name + code
|
|
103
|
+
self.file_text = Bm25Index() # file documents: path + all its code
|
|
104
|
+
self.by_id: dict[str, Chunk] = {}
|
|
105
|
+
self.embedder = None # optional: object with similarities(query, chunks) -> {id: cos}
|
|
106
|
+
self._version = 0
|
|
107
|
+
self._cache_dirty = False
|
|
108
|
+
self._graph: GraphIndex | None = None
|
|
109
|
+
self._callers: dict[str, list[str]] = {}
|
|
110
|
+
self._graph_version = -1
|
|
111
|
+
self._rebuild: threading.Thread | None = None
|
|
112
|
+
self._lock = threading.Lock()
|
|
113
|
+
|
|
114
|
+
# ---------- building ----------
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def from_chunks(cls, chunks: list[Chunk]) -> "RepoIndex":
|
|
118
|
+
"""An index over already-parsed chunks (offline evaluation; no files, no refresh)."""
|
|
119
|
+
index = cls()
|
|
120
|
+
by_file: dict[str, list[Chunk]] = {}
|
|
121
|
+
for c in chunks:
|
|
122
|
+
by_file.setdefault(c.file_path, []).append(c)
|
|
123
|
+
for path in sorted(by_file):
|
|
124
|
+
index._set_file(path, FileEntry(0, 0, by_file[path][0].file_hash, by_file[path]))
|
|
125
|
+
return index
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def open(cls, root: str | Path, cache_dir: Path = DEFAULT_CACHE_DIR) -> "RepoIndex":
|
|
129
|
+
"""Load the cached index of this working tree (if any) and bring it up to date."""
|
|
130
|
+
root = Path(root)
|
|
131
|
+
if not root.is_dir():
|
|
132
|
+
raise FileNotFoundError(f"repository folder not found: {root}")
|
|
133
|
+
index = cls(root, _cache_file(root, cache_dir))
|
|
134
|
+
index._load()
|
|
135
|
+
index.refresh()
|
|
136
|
+
return index
|
|
137
|
+
|
|
138
|
+
def _load(self) -> None:
|
|
139
|
+
if self.cache_file is None or not self.cache_file.is_file():
|
|
140
|
+
return
|
|
141
|
+
try:
|
|
142
|
+
data = json.loads(self.cache_file.read_text(encoding="utf-8"))
|
|
143
|
+
if data.get("version") != INDEX_VERSION:
|
|
144
|
+
return
|
|
145
|
+
for path, entry in sorted(data["files"].items()):
|
|
146
|
+
chunks = [Chunk.model_validate(c) for c in entry["chunks"]]
|
|
147
|
+
self._set_file(path, FileEntry(entry["size"], entry["mtime_ns"],
|
|
148
|
+
entry["sha256"], chunks))
|
|
149
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
150
|
+
log.warning("index cache unreadable, rebuilding: %s", exc)
|
|
151
|
+
self.files.clear()
|
|
152
|
+
self.text, self.file_text, self.by_id = Bm25Index(), Bm25Index(), {}
|
|
153
|
+
|
|
154
|
+
def save(self) -> None:
|
|
155
|
+
if self.cache_file is None or not self._cache_dirty:
|
|
156
|
+
return
|
|
157
|
+
self.cache_file.parent.mkdir(parents=True, exist_ok=True)
|
|
158
|
+
data = {"version": INDEX_VERSION, "files": {
|
|
159
|
+
path: {"size": e.size, "mtime_ns": e.mtime_ns, "sha256": e.sha256,
|
|
160
|
+
"chunks": [c.model_dump(mode="json") for c in e.chunks]}
|
|
161
|
+
for path, e in sorted(self.files.items())}}
|
|
162
|
+
tmp = self.cache_file.with_suffix(".tmp")
|
|
163
|
+
tmp.write_text(json.dumps(data), encoding="utf-8")
|
|
164
|
+
tmp.replace(self.cache_file)
|
|
165
|
+
self._cache_dirty = False
|
|
166
|
+
|
|
167
|
+
def refresh(self) -> list[str]:
|
|
168
|
+
"""Re-parse files whose content changed in the working tree. Returns changed paths."""
|
|
169
|
+
if self.root is None:
|
|
170
|
+
return []
|
|
171
|
+
current = list_python_files(self.root)
|
|
172
|
+
changed = []
|
|
173
|
+
parser = None
|
|
174
|
+
for rel in current:
|
|
175
|
+
try:
|
|
176
|
+
st = (self.root / rel).stat()
|
|
177
|
+
entry = self.files.get(rel)
|
|
178
|
+
if entry and entry.size == st.st_size and entry.mtime_ns == st.st_mtime_ns:
|
|
179
|
+
continue
|
|
180
|
+
source = (self.root / rel).read_bytes()
|
|
181
|
+
except OSError as exc:
|
|
182
|
+
log.warning("cannot read %s: %s", rel, exc)
|
|
183
|
+
continue
|
|
184
|
+
sha = hashlib.sha256(source).hexdigest()
|
|
185
|
+
self._cache_dirty = True
|
|
186
|
+
if entry and entry.sha256 == sha: # touched, not changed
|
|
187
|
+
entry.size, entry.mtime_ns = st.st_size, st.st_mtime_ns
|
|
188
|
+
continue
|
|
189
|
+
parser = parser or PythonParser()
|
|
190
|
+
self._set_file(rel, FileEntry(st.st_size, st.st_mtime_ns, sha,
|
|
191
|
+
parser.parse_file(rel, source)))
|
|
192
|
+
changed.append(rel)
|
|
193
|
+
for rel in sorted(set(self.files) - set(current)):
|
|
194
|
+
self._drop_file(rel)
|
|
195
|
+
changed.append(rel)
|
|
196
|
+
if changed:
|
|
197
|
+
self._version += 1
|
|
198
|
+
self._cache_dirty = True
|
|
199
|
+
return sorted(changed)
|
|
200
|
+
|
|
201
|
+
def _drop_file(self, path: str) -> None:
|
|
202
|
+
entry = self.files.pop(path, None)
|
|
203
|
+
if entry is None:
|
|
204
|
+
return
|
|
205
|
+
for c in entry.chunks:
|
|
206
|
+
self.text.remove(c.id)
|
|
207
|
+
self.by_id.pop(c.id, None)
|
|
208
|
+
self.file_text.remove(path)
|
|
209
|
+
|
|
210
|
+
def _set_file(self, path: str, entry: FileEntry) -> None:
|
|
211
|
+
self._drop_file(path)
|
|
212
|
+
self.files[path] = entry
|
|
213
|
+
words = tokenize(path)
|
|
214
|
+
for c in entry.chunks:
|
|
215
|
+
self.by_id[c.id] = c
|
|
216
|
+
self.text.add(c.id, tokenize(f"{c.file_path} {c.qualified_name} {c.code}"))
|
|
217
|
+
words.extend(tokenize(c.code))
|
|
218
|
+
self.file_text.add(path, words)
|
|
219
|
+
self._version += 1
|
|
220
|
+
|
|
221
|
+
def chunks(self) -> list[Chunk]:
|
|
222
|
+
return [c for path in sorted(self.files) for c in self.files[path].chunks]
|
|
223
|
+
|
|
224
|
+
# ---------- code graph ----------
|
|
225
|
+
|
|
226
|
+
def _build_graph(self, version: int) -> None:
|
|
227
|
+
graph = build_graph(self.chunks())
|
|
228
|
+
callers: dict[str, list[str]] = {}
|
|
229
|
+
for edge in graph.edges:
|
|
230
|
+
if edge.kind == EdgeKind.CALLS:
|
|
231
|
+
callers.setdefault(edge.target, []).append(edge.source)
|
|
232
|
+
with self._lock:
|
|
233
|
+
self._graph, self._callers, self._graph_version = GraphIndex(graph), callers, version
|
|
234
|
+
|
|
235
|
+
def graph(self, wait: bool = False) -> GraphIndex | None:
|
|
236
|
+
"""The code graph; after a change the previous graph is returned while a new one is
|
|
237
|
+
built in the background (wait=True builds it now)."""
|
|
238
|
+
version = self._version
|
|
239
|
+
if self._graph_version == version:
|
|
240
|
+
return self._graph
|
|
241
|
+
if self._graph is None or wait:
|
|
242
|
+
self._build_graph(version)
|
|
243
|
+
return self._graph
|
|
244
|
+
current = self._graph
|
|
245
|
+
if self._rebuild is None or not self._rebuild.is_alive():
|
|
246
|
+
self._rebuild = threading.Thread(target=self._build_graph, args=(version,),
|
|
247
|
+
daemon=True)
|
|
248
|
+
self._rebuild.start()
|
|
249
|
+
return current
|
|
250
|
+
|
|
251
|
+
# ---------- search ----------
|
|
252
|
+
|
|
253
|
+
def search(self, query: str, k: int = 10, weights: tuple[float, ...] | None = None,
|
|
254
|
+
chunks: list[Chunk] | None = None) -> list[Hit]:
|
|
255
|
+
"""Best k chunks for the query. `weights`: (structural, text, file, embed,
|
|
256
|
+
test factor). `chunks`: precomputed self.chunks() (speed)."""
|
|
257
|
+
w_struct, w_text, w_file, w_embed, test_factor = weights or (
|
|
258
|
+
W_STRUCTURAL, W_TEXT, W_FILE, W_EMBED, TEST_FACTOR)
|
|
259
|
+
chunks = chunks if chunks is not None else self.chunks()
|
|
260
|
+
words = tokenize(query)
|
|
261
|
+
text = normalised(self.text.scores(words))
|
|
262
|
+
file_text = normalised(self.file_text.scores(words))
|
|
263
|
+
mentions = analyse_query(query, chunks).weights
|
|
264
|
+
if not mentions:
|
|
265
|
+
best = sorted(text, key=lambda cid: (-text[cid], cid))[:PSEUDO_MENTIONS]
|
|
266
|
+
mentions = {cid: SYMBOL_WEIGHT for cid in best}
|
|
267
|
+
graph = self.graph()
|
|
268
|
+
if graph is not None:
|
|
269
|
+
structural = structural_scores(graph, mentions)
|
|
270
|
+
else:
|
|
271
|
+
structural = dict(mentions)
|
|
272
|
+
top = max(structural.values(), default=0.0)
|
|
273
|
+
structural = {cid: s / top for cid, s in structural.items()} if top > 0 else {}
|
|
274
|
+
embed: dict[str, float] = {}
|
|
275
|
+
if self.embedder is not None and w_embed > 0:
|
|
276
|
+
sims = self.embedder.similarities(query, chunks)
|
|
277
|
+
if sims:
|
|
278
|
+
lo, hi = min(sims.values()), max(sims.values())
|
|
279
|
+
embed = {cid: (s - lo) / (hi - lo) for cid, s in sims.items()} if hi > lo else {}
|
|
280
|
+
candidates = set(text) | set(structural) | {c.id for c in chunks
|
|
281
|
+
if c.file_path in file_text}
|
|
282
|
+
candidates |= set(sorted(embed, key=lambda cid: -embed[cid])[:500])
|
|
283
|
+
hits = []
|
|
284
|
+
for cid in candidates:
|
|
285
|
+
chunk = self.by_id.get(cid)
|
|
286
|
+
if chunk is None:
|
|
287
|
+
continue
|
|
288
|
+
s, t = structural.get(cid, 0.0), text.get(cid, 0.0)
|
|
289
|
+
f, e = file_text.get(chunk.file_path, 0.0), embed.get(cid, 0.0)
|
|
290
|
+
score = w_struct * s + w_text * t + w_file * f + (w_embed * e if embed else 0.0)
|
|
291
|
+
if is_test_file(chunk.file_path):
|
|
292
|
+
score *= test_factor
|
|
293
|
+
hits.append(Hit(chunk, score, s, t, f, e))
|
|
294
|
+
hits.sort(key=lambda h: (-h.score, h.chunk.id))
|
|
295
|
+
return hits[:k]
|
|
296
|
+
|
|
297
|
+
# ---------- what the agent sees ----------
|
|
298
|
+
|
|
299
|
+
def find_code(self, query: str, k: int = 8, max_tokens: int = 3000) -> str:
|
|
300
|
+
hits = self.search(query, k)
|
|
301
|
+
if not hits:
|
|
302
|
+
return "No matching code found."
|
|
303
|
+
out = [(f"{len(hits)} best matches (best first). Code is shown while it fits "
|
|
304
|
+
f"{max_tokens} tokens; use `symbol` for the rest.")]
|
|
305
|
+
used = count_tokens(out[0])
|
|
306
|
+
for i, hit in enumerate(hits, 1):
|
|
307
|
+
c = hit.chunk
|
|
308
|
+
head = f"\n{i}. {breadcrumb(c)} (lines {c.start_line}-{c.end_line})"
|
|
309
|
+
if c.kind != ChunkKind.MODULE and c.signature:
|
|
310
|
+
head += f"\n {_signature_line(c)}"
|
|
311
|
+
code = numbered_code(c)
|
|
312
|
+
cost = count_tokens(head) + count_tokens(code)
|
|
313
|
+
if used + cost <= max_tokens:
|
|
314
|
+
out += [head, code]
|
|
315
|
+
used += cost
|
|
316
|
+
else:
|
|
317
|
+
out.append(head)
|
|
318
|
+
used += count_tokens(head)
|
|
319
|
+
return "\n".join(out)
|
|
320
|
+
|
|
321
|
+
def _file_importance(self) -> dict[str, int]:
|
|
322
|
+
"""Number of other non-test files whose code points at each file (calls / imports /
|
|
323
|
+
bases). Tests are left out: they would make test helpers look most important."""
|
|
324
|
+
graph = self.graph()
|
|
325
|
+
if graph is None:
|
|
326
|
+
return {}
|
|
327
|
+
users: dict[str, set[str]] = {}
|
|
328
|
+
for source, targets in graph.undirected.items():
|
|
329
|
+
src_file = file_of(source)
|
|
330
|
+
if is_test_file(src_file):
|
|
331
|
+
continue
|
|
332
|
+
for target in targets:
|
|
333
|
+
tgt_file = file_of(target)
|
|
334
|
+
if tgt_file != src_file:
|
|
335
|
+
users.setdefault(tgt_file, set()).add(src_file)
|
|
336
|
+
return {path: len(files) for path, files in users.items()}
|
|
337
|
+
|
|
338
|
+
def repo_map(self, prefix: str = "", max_tokens: int = 2000) -> str:
|
|
339
|
+
"""Directory layout, then class / function signatures of the most-used files."""
|
|
340
|
+
prefix = prefix.strip("/")
|
|
341
|
+
paths = [p for p in sorted(self.files)
|
|
342
|
+
if not prefix or p == prefix or p.startswith(prefix + "/")]
|
|
343
|
+
if not paths:
|
|
344
|
+
return f"No Python files under '{prefix}'."
|
|
345
|
+
out = [f"# {len(paths)} Python files" + (f" under {prefix}/" if prefix else ""),
|
|
346
|
+
"## Directories (Python files below each)"]
|
|
347
|
+
out += _directory_lines(paths)
|
|
348
|
+
out.append("## Most-used files: classes and functions")
|
|
349
|
+
importance = self._file_importance()
|
|
350
|
+
ranked = sorted(paths, key=lambda p: (is_test_file(p), -importance.get(p, 0), p))
|
|
351
|
+
used = count_tokens("\n".join(out))
|
|
352
|
+
shown = 0
|
|
353
|
+
for path in ranked:
|
|
354
|
+
block = self._file_outline(path, importance.get(path, 0))
|
|
355
|
+
cost = count_tokens(block)
|
|
356
|
+
if used + cost > max_tokens:
|
|
357
|
+
continue
|
|
358
|
+
out.append(block)
|
|
359
|
+
used += cost
|
|
360
|
+
shown += 1
|
|
361
|
+
if shown < len(paths):
|
|
362
|
+
out.append(f"... {len(paths) - shown} more files not shown (pass a prefix, "
|
|
363
|
+
"or use find_code)")
|
|
364
|
+
return "\n".join(out)
|
|
365
|
+
|
|
366
|
+
def _file_outline(self, path: str, users: int) -> str:
|
|
367
|
+
chunks = self.files[path].chunks
|
|
368
|
+
lines = [f"{path}" + (f" (used by {users} files)" if users else "")]
|
|
369
|
+
methods: dict[str, list[str]] = {}
|
|
370
|
+
for c in chunks:
|
|
371
|
+
if c.kind == ChunkKind.METHOD and c.parent_id:
|
|
372
|
+
methods.setdefault(c.parent_id, []).append(c.qualified_name.split(".")[-1])
|
|
373
|
+
for c in chunks:
|
|
374
|
+
if c.kind == ChunkKind.CLASS and "." not in c.qualified_name:
|
|
375
|
+
names = methods.get(c.id, [])
|
|
376
|
+
listed = ", ".join(names[:MAP_METHODS_SHOWN])
|
|
377
|
+
more = f", +{len(names) - MAP_METHODS_SHOWN}" if len(names) > MAP_METHODS_SHOWN else ""
|
|
378
|
+
lines.append(f" {_signature_line(c)}" + (f": {listed}{more}" if names else ""))
|
|
379
|
+
elif c.kind == ChunkKind.FUNCTION:
|
|
380
|
+
lines.append(f" {_signature_line(c)}")
|
|
381
|
+
if len(lines) > MAP_LINES_PER_FILE + 1:
|
|
382
|
+
hidden = len(lines) - 1 - MAP_LINES_PER_FILE
|
|
383
|
+
lines = lines[: MAP_LINES_PER_FILE + 1] + [f" ... +{hidden} more (use symbol / find_code)"]
|
|
384
|
+
return "\n".join(lines)
|
|
385
|
+
|
|
386
|
+
def _match(self, name: str) -> list[Chunk]:
|
|
387
|
+
name = name.strip().replace("::", ".")
|
|
388
|
+
exact = [c for c in self.by_id.values()
|
|
389
|
+
if c.kind != ChunkKind.MODULE and (c.qualified_name == name or c.id == name
|
|
390
|
+
or c.qualified_name.endswith("." + name))]
|
|
391
|
+
if not exact:
|
|
392
|
+
last = name.split(".")[-1].lower()
|
|
393
|
+
exact = [c for c in self.by_id.values() if c.kind != ChunkKind.MODULE
|
|
394
|
+
and c.qualified_name.split(".")[-1].lower() == last]
|
|
395
|
+
return sorted(exact, key=lambda c: (c.file_path, c.start_line, c.id))
|
|
396
|
+
|
|
397
|
+
def symbol(self, name: str) -> str:
|
|
398
|
+
matches = self._match(name)
|
|
399
|
+
if not matches:
|
|
400
|
+
return f"No symbol named '{name}'. Try find_code."
|
|
401
|
+
if len(matches) > MAX_SYMBOL_MATCHES:
|
|
402
|
+
listed = "\n".join(f"- {breadcrumb(c)} (lines {c.start_line}-{c.end_line})"
|
|
403
|
+
for c in matches[:40])
|
|
404
|
+
more = f"\n... {len(matches) - 40} more" if len(matches) > 40 else ""
|
|
405
|
+
return (f"{len(matches)} symbols match '{name}'; ask again with a qualified name "
|
|
406
|
+
f"(e.g. Class.method):\n{listed}{more}")
|
|
407
|
+
graph = self.graph()
|
|
408
|
+
blocks = []
|
|
409
|
+
for c in matches:
|
|
410
|
+
block = [f"{breadcrumb(c)} (lines {c.start_line}-{c.end_line})", numbered_code(c)]
|
|
411
|
+
callers = sorted(set(self._callers.get(c.id, [])))
|
|
412
|
+
if callers:
|
|
413
|
+
block.append(_related("called by", callers))
|
|
414
|
+
if graph is not None and graph.depends_on.get(c.id):
|
|
415
|
+
block.append(_related("calls / inherits", sorted(set(graph.depends_on[c.id]))))
|
|
416
|
+
blocks.append("\n".join(block))
|
|
417
|
+
return "\n\n".join(blocks)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def is_test_file(path: str) -> bool:
|
|
421
|
+
parts = PurePosixPath(path).parts
|
|
422
|
+
name = parts[-1] if parts else ""
|
|
423
|
+
return (name.startswith("test_") or name.endswith("_test.py") or name == "conftest.py"
|
|
424
|
+
or any(part in ("tests", "test", "testing") for part in parts[:-1]))
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _directory_lines(paths: list[str]) -> list[str]:
|
|
428
|
+
"""Directories with the number of Python files below them, at the deepest level that
|
|
429
|
+
fits MAP_DIR_LINES lines (deep trees such as 100 locale folders collapse into one)."""
|
|
430
|
+
lines: list[str] = []
|
|
431
|
+
for depth in range(4, 0, -1):
|
|
432
|
+
counts: dict[str, int] = {}
|
|
433
|
+
for p in paths:
|
|
434
|
+
parts = PurePosixPath(p).parts[:-1]
|
|
435
|
+
key = "/".join(parts[:depth]) if parts else "."
|
|
436
|
+
counts[key] = counts.get(key, 0) + 1
|
|
437
|
+
lines = [f"{d}/ ({counts[d]})" for d in sorted(counts)]
|
|
438
|
+
if len(lines) <= MAP_DIR_LINES:
|
|
439
|
+
return lines
|
|
440
|
+
return lines[:MAP_DIR_LINES] + [f"... {len(lines) - MAP_DIR_LINES} more (pass a prefix)"]
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _related(label: str, ids: list[str]) -> str:
|
|
444
|
+
shown = ", ".join(i.replace("::", ":") for i in ids[:MAX_RELATED])
|
|
445
|
+
more = f" (+{len(ids) - MAX_RELATED} more)" if len(ids) > MAX_RELATED else ""
|
|
446
|
+
return f"{label}: {shown}{more}"
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def timed(fn: Callable[[], str]) -> str:
|
|
450
|
+
"""Run a tool function; any failure becomes a short message instead of a crash."""
|
|
451
|
+
try:
|
|
452
|
+
return fn()
|
|
453
|
+
except Exception as exc: # the agent gets an answer, the server keeps running
|
|
454
|
+
log.exception("index tool failed")
|
|
455
|
+
return f"TokenCut index error: {type(exc).__name__}: {exc}"
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""The repo index as a local MCP server (Model Context Protocol, stdio transport).
|
|
2
|
+
|
|
3
|
+
claude mcp add tokencut -- python -m tokencut.repoindex serve --repo .
|
|
4
|
+
|
|
5
|
+
The agent then has three tools (Claude Code names them mcp__tokencut__<name>). Every call
|
|
6
|
+
first refreshes the index from the working tree, so answers include uncommitted edits.
|
|
7
|
+
Messages are JSON-RPC 2.0, one per line on stdin / stdout; logs go to stderr.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
from typing import IO
|
|
15
|
+
|
|
16
|
+
from tokencut.repoindex.index import RepoIndex, timed
|
|
17
|
+
|
|
18
|
+
log = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
SERVER_INFO = {"name": "tokencut-repoindex", "version": "0.1.0"}
|
|
21
|
+
DEFAULT_PROTOCOL = "2025-06-18"
|
|
22
|
+
SAVE_EVERY_SECONDS = 60
|
|
23
|
+
|
|
24
|
+
TOOLS = [
|
|
25
|
+
{
|
|
26
|
+
"name": "repo_map",
|
|
27
|
+
"description": (
|
|
28
|
+
"Overview of the repository: directories, then the classes and function "
|
|
29
|
+
"signatures of its most-used files. Call it first in an unfamiliar repository, "
|
|
30
|
+
"or with a directory prefix to zoom in. Cheaper than listing and reading files."
|
|
31
|
+
),
|
|
32
|
+
"inputSchema": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"properties": {
|
|
35
|
+
"prefix": {"type": "string", "description": "Only files under this directory"},
|
|
36
|
+
"max_tokens": {"type": "integer", "description": "Size limit (default 2000)"},
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"name": "find_code",
|
|
42
|
+
"description": (
|
|
43
|
+
"Find the functions / classes relevant to a question, bug report, error message "
|
|
44
|
+
"or traceback. Returns the best matches with file, line range and code (with line "
|
|
45
|
+
"numbers). Use it instead of several grep + read rounds."
|
|
46
|
+
),
|
|
47
|
+
"inputSchema": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"properties": {
|
|
50
|
+
"query": {"type": "string", "description": "Issue text, error, or symbol names"},
|
|
51
|
+
"k": {"type": "integer", "description": "Number of matches (default 8)"},
|
|
52
|
+
"max_tokens": {"type": "integer", "description": "Code shown (default 3000)"},
|
|
53
|
+
},
|
|
54
|
+
"required": ["query"],
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"name": "symbol",
|
|
59
|
+
"description": (
|
|
60
|
+
"Exact current source of a function, method or class by name (e.g. "
|
|
61
|
+
"'QuerySet.only'), with line numbers, plus what calls it and what it calls."
|
|
62
|
+
),
|
|
63
|
+
"inputSchema": {
|
|
64
|
+
"type": "object",
|
|
65
|
+
"properties": {"name": {"type": "string", "description": "Name or Class.method"}},
|
|
66
|
+
"required": ["name"],
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _int(args: dict, key: str, default: int, lo: int, hi: int) -> int:
|
|
73
|
+
try:
|
|
74
|
+
return max(lo, min(hi, int(args.get(key, default))))
|
|
75
|
+
except (TypeError, ValueError):
|
|
76
|
+
return default
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def call_tool(index: RepoIndex, name: str, args: dict) -> tuple[str, bool]:
|
|
80
|
+
"""(text, is_error) for one tool call."""
|
|
81
|
+
index.refresh()
|
|
82
|
+
if name == "repo_map":
|
|
83
|
+
return timed(lambda: index.repo_map(str(args.get("prefix", "")),
|
|
84
|
+
_int(args, "max_tokens", 2000, 200, 20_000))), False
|
|
85
|
+
if name == "find_code":
|
|
86
|
+
query = str(args.get("query", "")).strip()
|
|
87
|
+
if not query:
|
|
88
|
+
return "find_code needs a query.", True
|
|
89
|
+
return timed(lambda: index.find_code(query, _int(args, "k", 8, 1, 50),
|
|
90
|
+
_int(args, "max_tokens", 3000, 200, 20_000))), False
|
|
91
|
+
if name == "symbol":
|
|
92
|
+
symbol = str(args.get("name", "")).strip()
|
|
93
|
+
if not symbol:
|
|
94
|
+
return "symbol needs a name.", True
|
|
95
|
+
return timed(lambda: index.symbol(symbol)), False
|
|
96
|
+
return f"Unknown tool {name!r}.", True
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def handle(index: RepoIndex, message: dict) -> dict | None:
|
|
100
|
+
"""Answer one JSON-RPC message (None for notifications)."""
|
|
101
|
+
method = message.get("method")
|
|
102
|
+
msg_id = message.get("id")
|
|
103
|
+
params = message.get("params") or {}
|
|
104
|
+
if msg_id is None:
|
|
105
|
+
return None # notification (e.g. notifications/initialized)
|
|
106
|
+
if method == "initialize":
|
|
107
|
+
result = {
|
|
108
|
+
"protocolVersion": params.get("protocolVersion") or DEFAULT_PROTOCOL,
|
|
109
|
+
"capabilities": {"tools": {"listChanged": False}},
|
|
110
|
+
"serverInfo": SERVER_INFO,
|
|
111
|
+
}
|
|
112
|
+
elif method == "ping":
|
|
113
|
+
result = {}
|
|
114
|
+
elif method == "tools/list":
|
|
115
|
+
result = {"tools": TOOLS}
|
|
116
|
+
elif method == "tools/call":
|
|
117
|
+
text, is_error = call_tool(index, str(params.get("name", "")),
|
|
118
|
+
params.get("arguments") or {})
|
|
119
|
+
result = {"content": [{"type": "text", "text": text}], "isError": is_error}
|
|
120
|
+
else:
|
|
121
|
+
return {"jsonrpc": "2.0", "id": msg_id,
|
|
122
|
+
"error": {"code": -32601, "message": f"Method not found: {method}"}}
|
|
123
|
+
return {"jsonrpc": "2.0", "id": msg_id, "result": result}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def serve(index: RepoIndex, stdin: IO[str] = sys.stdin, stdout: IO[str] = sys.stdout) -> None:
|
|
127
|
+
last_save = time.monotonic()
|
|
128
|
+
for line in stdin:
|
|
129
|
+
line = line.strip()
|
|
130
|
+
if not line:
|
|
131
|
+
continue
|
|
132
|
+
try:
|
|
133
|
+
message = json.loads(line)
|
|
134
|
+
except ValueError:
|
|
135
|
+
reply = {"jsonrpc": "2.0", "id": None,
|
|
136
|
+
"error": {"code": -32700, "message": "Parse error"}}
|
|
137
|
+
else:
|
|
138
|
+
reply = handle(index, message) if isinstance(message, dict) else None
|
|
139
|
+
if reply is not None:
|
|
140
|
+
stdout.write(json.dumps(reply) + "\n")
|
|
141
|
+
stdout.flush()
|
|
142
|
+
if time.monotonic() - last_save > SAVE_EVERY_SECONDS:
|
|
143
|
+
index.save()
|
|
144
|
+
last_save = time.monotonic()
|
|
145
|
+
index.save()
|
tokencut/tokens.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""The ONE place where tokens are counted. Swap the tokenizer here only."""
|
|
2
|
+
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
|
|
5
|
+
import tiktoken
|
|
6
|
+
|
|
7
|
+
ENCODING_NAME = "o200k_base" # proxy tokenizer for all providers in Iteration 1
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@lru_cache(maxsize=1)
|
|
11
|
+
def _encoding() -> tiktoken.Encoding:
|
|
12
|
+
# First call downloads the encoding file once, then it is cached on disk.
|
|
13
|
+
return tiktoken.get_encoding(ENCODING_NAME)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def count_tokens(text: str) -> int:
|
|
17
|
+
if not text:
|
|
18
|
+
return 0
|
|
19
|
+
return len(_encoding().encode(text, disallowed_special=()))
|
|
File without changes
|