tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"""Member B — Code-graph construction."""
|
|
2
|
+
|
|
3
|
+
import builtins
|
|
4
|
+
|
|
5
|
+
from tree_sitter import Node
|
|
6
|
+
|
|
7
|
+
from tokencut.contracts import Chunk, ChunkKind, CodeGraph, Edge, EdgeKind
|
|
8
|
+
from tokencut.graph.modules import ModuleMap, node_text, parse_code, read_imports
|
|
9
|
+
|
|
10
|
+
BUILTIN_NAMES = frozenset(dir(builtins))
|
|
11
|
+
|
|
12
|
+
# One entry of a file's name table: ("module", dotted name) or ("chunk", chunk id).
|
|
13
|
+
Binding = tuple[str, str]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def build_graph(chunks: list[Chunk]) -> CodeGraph:
|
|
17
|
+
"""Build contains / imports / calls / inherits edges between chunks.
|
|
18
|
+
|
|
19
|
+
Must:
|
|
20
|
+
- never point an edge at a chunk id that does not exist
|
|
21
|
+
- mark heuristic call/import guesses with resolved=False
|
|
22
|
+
- handle circular imports without infinite loops
|
|
23
|
+
"""
|
|
24
|
+
return _GraphBuilder(chunks).build()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def dependency_closure(graph: CodeGraph, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
|
|
28
|
+
"""Return chunk_ids plus everything they depend on within max_hops."""
|
|
29
|
+
from tokencut.graph.queries import GraphIndex # local import: queries imports nothing from here
|
|
30
|
+
|
|
31
|
+
return GraphIndex(graph).closure(chunk_ids, max_hops)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _GraphBuilder:
|
|
35
|
+
def __init__(self, chunks: list[Chunk]):
|
|
36
|
+
self.chunks = sorted(chunks, key=lambda c: (c.file_path, c.start_line, c.id))
|
|
37
|
+
self.by_id = {c.id: c for c in self.chunks}
|
|
38
|
+
self.modules = ModuleMap.build(self.chunks)
|
|
39
|
+
self.edges: dict[tuple[str, str, EdgeKind], bool] = {} # (source, target, kind) -> resolved
|
|
40
|
+
self.name_tables: dict[str, dict[str, Binding]] = {} # file path -> name table
|
|
41
|
+
|
|
42
|
+
# (class id, method name) -> method id; first definition wins (property getter before setter)
|
|
43
|
+
self.methods: dict[tuple[str, str], str] = {}
|
|
44
|
+
# last name -> chunk ids, for DECISIONS #7 rule 4
|
|
45
|
+
self.by_last_name: dict[str, list[str]] = {}
|
|
46
|
+
self.classes_by_name: dict[str, list[str]] = {}
|
|
47
|
+
for c in self.chunks:
|
|
48
|
+
if c.kind == ChunkKind.METHOD and c.parent_id:
|
|
49
|
+
self.methods.setdefault((c.parent_id, c.qualified_name.split(".")[-1]), c.id)
|
|
50
|
+
if c.kind != ChunkKind.MODULE:
|
|
51
|
+
last = c.qualified_name.split(".")[-1]
|
|
52
|
+
self.by_last_name.setdefault(last, []).append(c.id)
|
|
53
|
+
if c.kind == ChunkKind.CLASS:
|
|
54
|
+
self.classes_by_name.setdefault(last, []).append(c.id)
|
|
55
|
+
self.bases: dict[str, list[str]] = {} # class id -> resolved base class ids
|
|
56
|
+
|
|
57
|
+
# ---------- edge helpers ----------
|
|
58
|
+
|
|
59
|
+
def _add(self, source: str, target: str | None, kind: EdgeKind, resolved: bool = True) -> None:
|
|
60
|
+
if (
|
|
61
|
+
target is None
|
|
62
|
+
or source == target
|
|
63
|
+
or target not in self.by_id
|
|
64
|
+
or source not in self.by_id
|
|
65
|
+
):
|
|
66
|
+
return
|
|
67
|
+
key = (source, target, kind)
|
|
68
|
+
self.edges[key] = self.edges.get(key, False) or resolved
|
|
69
|
+
|
|
70
|
+
def build(self) -> CodeGraph:
|
|
71
|
+
for c in self.chunks:
|
|
72
|
+
if c.kind == ChunkKind.MODULE:
|
|
73
|
+
self._imports(c)
|
|
74
|
+
for c in self.chunks:
|
|
75
|
+
if c.kind == ChunkKind.METHOD and c.parent_id:
|
|
76
|
+
self._add(c.parent_id, c.id, EdgeKind.CONTAINS)
|
|
77
|
+
elif c.kind == ChunkKind.CLASS:
|
|
78
|
+
self._inherits(c)
|
|
79
|
+
for c in self.chunks: # after INHERITS, because self.foo() may resolve into a base class
|
|
80
|
+
if c.kind in (ChunkKind.FUNCTION, ChunkKind.METHOD):
|
|
81
|
+
self._calls(c)
|
|
82
|
+
|
|
83
|
+
edges = [
|
|
84
|
+
Edge(source=s, target=t, kind=k, resolved=r)
|
|
85
|
+
for (s, t, k), r in sorted(
|
|
86
|
+
self.edges.items(), key=lambda kv: (kv[0][0], kv[0][1], kv[0][2].value)
|
|
87
|
+
)
|
|
88
|
+
]
|
|
89
|
+
return CodeGraph(chunk_ids=[c.id for c in self.chunks], edges=edges)
|
|
90
|
+
|
|
91
|
+
# ---------- IMPORTS (plan B4) ----------
|
|
92
|
+
|
|
93
|
+
def _imports(self, module: Chunk) -> None:
|
|
94
|
+
table: dict[str, Binding] = {}
|
|
95
|
+
self.name_tables[module.file_path] = table
|
|
96
|
+
if not module.code.strip():
|
|
97
|
+
return
|
|
98
|
+
mm = self.modules
|
|
99
|
+
package = mm.package_of(module.file_path)
|
|
100
|
+
for imp in read_imports(parse_code(module.code), package):
|
|
101
|
+
if not imp.module:
|
|
102
|
+
continue
|
|
103
|
+
if imp.name is None and not imp.wildcard:
|
|
104
|
+
# import a.b / import a.b as z
|
|
105
|
+
if mm.is_module(imp.module):
|
|
106
|
+
self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
|
|
107
|
+
if imp.alias:
|
|
108
|
+
if mm.is_module(imp.module):
|
|
109
|
+
table[imp.alias] = ("module", imp.module)
|
|
110
|
+
else:
|
|
111
|
+
first = imp.module.split(".")[0]
|
|
112
|
+
if mm.is_module(first):
|
|
113
|
+
table[first] = ("module", first)
|
|
114
|
+
elif imp.wildcard:
|
|
115
|
+
if mm.is_module(imp.module):
|
|
116
|
+
self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
|
|
117
|
+
else:
|
|
118
|
+
# from a.b import y [as z]
|
|
119
|
+
local = imp.alias or imp.name
|
|
120
|
+
submodule = f"{imp.module}.{imp.name}"
|
|
121
|
+
if mm.is_module(submodule):
|
|
122
|
+
self._add(module.id, mm.module_chunk[submodule], EdgeKind.IMPORTS)
|
|
123
|
+
table[local] = ("module", submodule)
|
|
124
|
+
elif mm.is_module(imp.module):
|
|
125
|
+
self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
|
|
126
|
+
target = mm.top_names[imp.module].get(imp.name)
|
|
127
|
+
if target is not None:
|
|
128
|
+
table[local] = ("chunk", target)
|
|
129
|
+
# external module: ignored
|
|
130
|
+
|
|
131
|
+
# ---------- name resolution (DECISIONS #7) ----------
|
|
132
|
+
|
|
133
|
+
def _same_file_name(self, file_path: str, name: str) -> str | None:
|
|
134
|
+
module = self.modules.file_module.get(file_path)
|
|
135
|
+
if module is None:
|
|
136
|
+
return None
|
|
137
|
+
return self.modules.top_names[module].get(name)
|
|
138
|
+
|
|
139
|
+
def _unique(self, name: str, candidates: dict[str, list[str]]) -> str | None:
|
|
140
|
+
ids = candidates.get(name, [])
|
|
141
|
+
return ids[0] if len(ids) == 1 else None
|
|
142
|
+
|
|
143
|
+
def _resolve_name(
|
|
144
|
+
self, file_path: str, name: str, classes_only: bool = False
|
|
145
|
+
) -> tuple[str | None, bool]:
|
|
146
|
+
"""Resolve a bare identifier. Returns (chunk id or None, resolved flag)."""
|
|
147
|
+
binding = self.name_tables.get(file_path, {}).get(name)
|
|
148
|
+
if binding is not None: # rule 1: imported name
|
|
149
|
+
if binding[0] == "chunk" and self._kind_ok(binding[1], classes_only):
|
|
150
|
+
return binding[1], True
|
|
151
|
+
return None, True # a module, or the wrong kind: no edge, and no guessing
|
|
152
|
+
local = self._same_file_name(file_path, name) # rule 3: same file
|
|
153
|
+
if local is not None and self._kind_ok(local, classes_only):
|
|
154
|
+
return local, True
|
|
155
|
+
if name in BUILTIN_NAMES:
|
|
156
|
+
return None, True
|
|
157
|
+
return self._rule4(name, classes_only)
|
|
158
|
+
|
|
159
|
+
def _rule4(self, name: str, classes_only: bool = False) -> tuple[str | None, bool]:
|
|
160
|
+
# rule 4: exactly one definition with this name in the whole repo -> guess, resolved=False
|
|
161
|
+
target = self._unique(name, self.classes_by_name if classes_only else self.by_last_name)
|
|
162
|
+
return target, False
|
|
163
|
+
|
|
164
|
+
def _kind_ok(self, chunk_id: str, classes_only: bool) -> bool:
|
|
165
|
+
return not classes_only or self.by_id[chunk_id].kind == ChunkKind.CLASS
|
|
166
|
+
|
|
167
|
+
def _module_of_expr(self, file_path: str, node: Node) -> str | None:
|
|
168
|
+
"""If node is `alias` or `alias.sub` naming an imported repo module, return its dotted name."""
|
|
169
|
+
if node.type == "identifier":
|
|
170
|
+
binding = self.name_tables.get(file_path, {}).get(node_text(node))
|
|
171
|
+
if binding is not None and binding[0] == "module":
|
|
172
|
+
return binding[1]
|
|
173
|
+
return None
|
|
174
|
+
if node.type == "attribute":
|
|
175
|
+
obj = node.child_by_field_name("object")
|
|
176
|
+
attr = node.child_by_field_name("attribute")
|
|
177
|
+
if obj is None or attr is None:
|
|
178
|
+
return None
|
|
179
|
+
parent = self._module_of_expr(file_path, obj)
|
|
180
|
+
if parent is not None and self.modules.is_module(f"{parent}.{node_text(attr)}"):
|
|
181
|
+
return f"{parent}.{node_text(attr)}"
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
def _method_in_class_or_bases(self, class_id: str, name: str) -> str | None:
|
|
185
|
+
target = self.methods.get((class_id, name))
|
|
186
|
+
if target is not None:
|
|
187
|
+
return target
|
|
188
|
+
for base in self.bases.get(class_id, []): # one level of base classes
|
|
189
|
+
target = self.methods.get((base, name))
|
|
190
|
+
if target is not None:
|
|
191
|
+
return target
|
|
192
|
+
return None
|
|
193
|
+
|
|
194
|
+
def _class_binding(self, file_path: str, name: str) -> str | None:
|
|
195
|
+
"""Chunk id if `name` is a class visible in this file (imported or defined here)."""
|
|
196
|
+
binding = self.name_tables.get(file_path, {}).get(name)
|
|
197
|
+
if binding is not None:
|
|
198
|
+
target = binding[1] if binding[0] == "chunk" else None
|
|
199
|
+
else:
|
|
200
|
+
target = self._same_file_name(file_path, name)
|
|
201
|
+
if target is not None and self.by_id[target].kind == ChunkKind.CLASS:
|
|
202
|
+
return target
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
# ---------- INHERITS (plan B3) ----------
|
|
206
|
+
|
|
207
|
+
def _inherits(self, cls: Chunk) -> None:
|
|
208
|
+
# The class signature holds the bases; give it a body so tree-sitter sees a full class.
|
|
209
|
+
root = parse_code(cls.signature + "\n pass\n")
|
|
210
|
+
definition = next((n for n in _walk(root) if n.type == "class_definition"), None)
|
|
211
|
+
if definition is None:
|
|
212
|
+
return
|
|
213
|
+
superclasses = definition.child_by_field_name("superclasses")
|
|
214
|
+
if superclasses is None:
|
|
215
|
+
return
|
|
216
|
+
resolved_bases = []
|
|
217
|
+
for base in superclasses.named_children:
|
|
218
|
+
if base.type == "keyword_argument": # metaclass=...
|
|
219
|
+
continue
|
|
220
|
+
if base.type == "subscript": # Base[T]
|
|
221
|
+
base = base.child_by_field_name("value") or base
|
|
222
|
+
target, resolved = self._resolve_base(cls.file_path, base)
|
|
223
|
+
if target is not None:
|
|
224
|
+
self._add(cls.id, target, EdgeKind.INHERITS, resolved)
|
|
225
|
+
resolved_bases.append(target)
|
|
226
|
+
self.bases[cls.id] = resolved_bases
|
|
227
|
+
|
|
228
|
+
def _resolve_base(self, file_path: str, base: Node) -> tuple[str | None, bool]:
|
|
229
|
+
if base.type == "identifier":
|
|
230
|
+
return self._resolve_name(file_path, node_text(base), classes_only=True)
|
|
231
|
+
if base.type == "attribute":
|
|
232
|
+
obj = base.child_by_field_name("object")
|
|
233
|
+
name = node_text(base.child_by_field_name("attribute"))
|
|
234
|
+
module = self._module_of_expr(file_path, obj) if obj is not None else None
|
|
235
|
+
if module is not None:
|
|
236
|
+
target = self.modules.top_names[module].get(name)
|
|
237
|
+
if target is not None and self._kind_ok(target, True):
|
|
238
|
+
return target, True
|
|
239
|
+
return self._rule4(name, classes_only=True)
|
|
240
|
+
return None, True
|
|
241
|
+
|
|
242
|
+
# ---------- CALLS (plan B5) ----------
|
|
243
|
+
|
|
244
|
+
def _calls(self, func: Chunk) -> None:
|
|
245
|
+
root = parse_code(func.code)
|
|
246
|
+
for node in _walk(root):
|
|
247
|
+
if node.type != "call":
|
|
248
|
+
continue
|
|
249
|
+
callee = node.child_by_field_name("function")
|
|
250
|
+
if callee is None:
|
|
251
|
+
continue
|
|
252
|
+
target, resolved = self._resolve_callee(func, callee)
|
|
253
|
+
self._add(func.id, target, EdgeKind.CALLS, resolved)
|
|
254
|
+
|
|
255
|
+
def _resolve_callee(self, func: Chunk, callee: Node) -> tuple[str | None, bool]:
|
|
256
|
+
file_path = func.file_path
|
|
257
|
+
if callee.type == "identifier":
|
|
258
|
+
return self._resolve_name(file_path, node_text(callee))
|
|
259
|
+
if callee.type != "attribute":
|
|
260
|
+
return None, True # e.g. f()() or x[0]()
|
|
261
|
+
obj = callee.child_by_field_name("object")
|
|
262
|
+
attr = callee.child_by_field_name("attribute")
|
|
263
|
+
if obj is None or attr is None:
|
|
264
|
+
return None, True
|
|
265
|
+
name = node_text(attr)
|
|
266
|
+
in_method = func.kind == ChunkKind.METHOD and func.parent_id is not None
|
|
267
|
+
|
|
268
|
+
# rule 2: self.foo() / cls.foo() inside a method -> same class, then its base classes
|
|
269
|
+
if in_method and obj.type == "identifier" and node_text(obj) in ("self", "cls"):
|
|
270
|
+
target = self._method_in_class_or_bases(func.parent_id, name)
|
|
271
|
+
if target is not None:
|
|
272
|
+
return target, True
|
|
273
|
+
# super().foo() -> base classes only
|
|
274
|
+
elif in_method and obj.type == "call" and _is_super_call(obj):
|
|
275
|
+
for base in self.bases.get(func.parent_id, []):
|
|
276
|
+
target = self.methods.get((base, name))
|
|
277
|
+
if target is not None:
|
|
278
|
+
return target, True
|
|
279
|
+
else:
|
|
280
|
+
# rule 1: module_alias.foo()
|
|
281
|
+
module = self._module_of_expr(file_path, obj)
|
|
282
|
+
if module is not None:
|
|
283
|
+
target = self.modules.top_names[module].get(name)
|
|
284
|
+
if target is not None:
|
|
285
|
+
return target, True
|
|
286
|
+
# rule 1 / 3: SomeClass.method() where the class is imported or defined in this file
|
|
287
|
+
elif obj.type == "identifier":
|
|
288
|
+
class_id = self._class_binding(file_path, node_text(obj))
|
|
289
|
+
if class_id is not None:
|
|
290
|
+
target = self._method_in_class_or_bases(class_id, name)
|
|
291
|
+
if target is not None:
|
|
292
|
+
return target, True
|
|
293
|
+
return self._rule4(name)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _is_super_call(node: Node) -> bool:
|
|
297
|
+
fn = node.child_by_field_name("function")
|
|
298
|
+
return fn is not None and fn.type == "identifier" and node_text(fn) == "super"
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _walk(root: Node):
|
|
302
|
+
"""All nodes under root in source order (explicit stack: no recursion limit on deep code)."""
|
|
303
|
+
stack = [root]
|
|
304
|
+
while stack:
|
|
305
|
+
node = stack.pop()
|
|
306
|
+
yield node
|
|
307
|
+
stack.extend(reversed(node.children))
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""File path <-> dotted module name, and reading import statements (IMPLEMENTATION_PLAN B2, B4)."""
|
|
2
|
+
|
|
3
|
+
import textwrap
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
import tree_sitter_python as tspython
|
|
7
|
+
from tree_sitter import Language, Node, Parser
|
|
8
|
+
|
|
9
|
+
from tokencut.contracts import Chunk, ChunkKind
|
|
10
|
+
|
|
11
|
+
_PARSER = Parser(Language(tspython.language()))
|
|
12
|
+
|
|
13
|
+
# Source-layout folders that are not part of the import name (pytest uses src/_pytest/...).
|
|
14
|
+
LAYOUT_PREFIXES = ("src", "lib")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_code(code: str) -> Node:
|
|
18
|
+
"""Parse a chunk's code with tree-sitter. Methods are indented, so dedent first."""
|
|
19
|
+
return _PARSER.parse(textwrap.dedent(code).encode("utf-8")).root_node
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def node_text(node: Node) -> str:
|
|
23
|
+
return node.text.decode("utf-8", errors="replace")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def module_name_for(file_path: str) -> str:
|
|
27
|
+
"""'shop/cart.py' -> 'shop.cart'; 'shop/payments/__init__.py' -> 'shop.payments'."""
|
|
28
|
+
parts = (
|
|
29
|
+
file_path[: -len(".py")].split("/") if file_path.endswith(".py") else file_path.split("/")
|
|
30
|
+
)
|
|
31
|
+
if parts and parts[-1] == "__init__" and len(parts) > 1:
|
|
32
|
+
parts = parts[:-1]
|
|
33
|
+
return ".".join(parts)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def is_package_file(file_path: str) -> bool:
|
|
37
|
+
return file_path.endswith("/__init__.py") or file_path == "__init__.py"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class ModuleMap:
|
|
42
|
+
"""Lookup tables over all module chunks of the repo."""
|
|
43
|
+
|
|
44
|
+
module_chunk: dict[str, str] = field(default_factory=dict) # module name -> module chunk id
|
|
45
|
+
file_module: dict[str, str] = field(default_factory=dict) # file path -> primary module name
|
|
46
|
+
top_names: dict[str, dict[str, str]] = field(
|
|
47
|
+
default_factory=dict
|
|
48
|
+
) # module -> {name -> chunk id}
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def build(cls, chunks: list[Chunk]) -> "ModuleMap":
|
|
52
|
+
mm = cls()
|
|
53
|
+
files = sorted({c.file_path for c in chunks if c.kind == ChunkKind.MODULE})
|
|
54
|
+
names_by_file: dict[str, dict[str, str]] = {f: {} for f in files}
|
|
55
|
+
for chunk in chunks: # chunks are in line order, so a later definition wins (as in Python)
|
|
56
|
+
if (
|
|
57
|
+
chunk.kind in (ChunkKind.FUNCTION, ChunkKind.CLASS)
|
|
58
|
+
and chunk.file_path in names_by_file
|
|
59
|
+
):
|
|
60
|
+
names_by_file[chunk.file_path][chunk.qualified_name] = chunk.id
|
|
61
|
+
|
|
62
|
+
# Primary names first, so a 'src/'-less alias never hides a real module of that name.
|
|
63
|
+
for file_path in files:
|
|
64
|
+
name = module_name_for(file_path)
|
|
65
|
+
mm.file_module[file_path] = name
|
|
66
|
+
mm.module_chunk[name] = f"{file_path}::<module>"
|
|
67
|
+
mm.top_names[name] = names_by_file[file_path]
|
|
68
|
+
for file_path in files:
|
|
69
|
+
name = mm.file_module[file_path]
|
|
70
|
+
first, _, rest = name.partition(".")
|
|
71
|
+
if first in LAYOUT_PREFIXES and rest and rest not in mm.module_chunk:
|
|
72
|
+
mm.module_chunk[rest] = mm.module_chunk[name]
|
|
73
|
+
mm.top_names[rest] = mm.top_names[name]
|
|
74
|
+
return mm
|
|
75
|
+
|
|
76
|
+
def is_module(self, name: str) -> bool:
|
|
77
|
+
return name in self.module_chunk
|
|
78
|
+
|
|
79
|
+
def package_of(self, file_path: str) -> str:
|
|
80
|
+
"""The package that relative imports in this file are resolved against."""
|
|
81
|
+
name = self.file_module[file_path]
|
|
82
|
+
if is_package_file(file_path):
|
|
83
|
+
return name
|
|
84
|
+
return name.rpartition(".")[0]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class ImportedName:
|
|
89
|
+
"""One name bound by an import statement."""
|
|
90
|
+
|
|
91
|
+
module: str # absolute dotted module the statement refers to ('' if unresolvable)
|
|
92
|
+
name: str | None # imported name for 'from m import name'; None for 'import m'
|
|
93
|
+
alias: str | None # local binding if 'as' was used
|
|
94
|
+
wildcard: bool = False
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def read_imports(module_root: Node, package: str) -> list[ImportedName]:
|
|
98
|
+
"""Collect import statements anywhere in a module chunk (e.g. also inside try/if blocks)."""
|
|
99
|
+
found: list[ImportedName] = []
|
|
100
|
+
stack = [module_root]
|
|
101
|
+
while stack:
|
|
102
|
+
node = stack.pop()
|
|
103
|
+
if node.type == "import_statement":
|
|
104
|
+
for child in node.children_by_field_name("name"):
|
|
105
|
+
dotted, alias = _name_and_alias(child)
|
|
106
|
+
found.append(ImportedName(module=dotted, name=None, alias=alias))
|
|
107
|
+
elif node.type == "import_from_statement":
|
|
108
|
+
module_node = node.child_by_field_name("module_name")
|
|
109
|
+
base = _resolve_from_module(module_node, package) if module_node is not None else ""
|
|
110
|
+
names = node.children_by_field_name("name")
|
|
111
|
+
if any(c.type == "wildcard_import" for c in node.children):
|
|
112
|
+
found.append(ImportedName(module=base, name=None, alias=None, wildcard=True))
|
|
113
|
+
for child in names:
|
|
114
|
+
dotted, alias = _name_and_alias(child)
|
|
115
|
+
found.append(ImportedName(module=base, name=dotted, alias=alias))
|
|
116
|
+
else:
|
|
117
|
+
stack.extend(reversed(node.children))
|
|
118
|
+
return found
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _name_and_alias(node: Node) -> tuple[str, str | None]:
|
|
122
|
+
if node.type == "aliased_import":
|
|
123
|
+
name_node = node.child_by_field_name("name")
|
|
124
|
+
alias_node = node.child_by_field_name("alias")
|
|
125
|
+
return (
|
|
126
|
+
node_text(name_node) if name_node else "",
|
|
127
|
+
node_text(alias_node) if alias_node else None,
|
|
128
|
+
)
|
|
129
|
+
return node_text(node), None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _resolve_from_module(module_node: Node, package: str) -> str:
|
|
133
|
+
"""'from ..a.b import x' inside package 'p.q.r' -> 'p.q.a.b'. '' if it escapes the repo."""
|
|
134
|
+
if module_node.type != "relative_import":
|
|
135
|
+
return node_text(module_node)
|
|
136
|
+
level = 0
|
|
137
|
+
rest = ""
|
|
138
|
+
for child in module_node.children:
|
|
139
|
+
if child.type == "import_prefix":
|
|
140
|
+
level = node_text(child).count(".")
|
|
141
|
+
elif child.type == "dotted_name":
|
|
142
|
+
rest = node_text(child)
|
|
143
|
+
parts = package.split(".") if package else []
|
|
144
|
+
if level - 1 > len(parts):
|
|
145
|
+
return ""
|
|
146
|
+
base_parts = parts[: len(parts) - (level - 1)]
|
|
147
|
+
if rest:
|
|
148
|
+
base_parts = base_parts + rest.split(".")
|
|
149
|
+
return ".".join(base_parts)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Read-only questions about a CodeGraph (IMPLEMENTATION_PLAN B6).
|
|
2
|
+
|
|
3
|
+
GraphIndex pre-computes adjacency lists once; the module-level functions are
|
|
4
|
+
convenience wrappers for one-off calls. Results are sets/dicts: callers must
|
|
5
|
+
sort before producing output (DECISIONS #8).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections import deque
|
|
9
|
+
|
|
10
|
+
import networkx as nx
|
|
11
|
+
|
|
12
|
+
from tokencut.contracts import CodeGraph, EdgeKind
|
|
13
|
+
|
|
14
|
+
MODULE_SUFFIX = "::<module>"
|
|
15
|
+
_DEPENDENCY_KINDS = (EdgeKind.CALLS, EdgeKind.INHERITS)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def file_of(chunk_id: str) -> str:
|
|
19
|
+
"""'shop/cart.py::Cart.total' -> 'shop/cart.py' (file paths never contain '::')."""
|
|
20
|
+
return chunk_id.split("::", 1)[0]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class GraphIndex:
|
|
24
|
+
def __init__(self, graph: CodeGraph):
|
|
25
|
+
self.chunk_ids = set(graph.chunk_ids)
|
|
26
|
+
self.depends_on: dict[str, list[str]] = {cid: [] for cid in graph.chunk_ids}
|
|
27
|
+
self.undirected: dict[str, list[str]] = {cid: [] for cid in graph.chunk_ids}
|
|
28
|
+
self.parent: dict[str, str] = {} # method id -> class id (from CONTAINS edges)
|
|
29
|
+
for edge in graph.edges: # graph.edges is sorted, so neighbour lists are deterministic
|
|
30
|
+
if edge.source not in self.chunk_ids or edge.target not in self.chunk_ids:
|
|
31
|
+
continue
|
|
32
|
+
if edge.kind in _DEPENDENCY_KINDS:
|
|
33
|
+
self.depends_on[edge.source].append(edge.target)
|
|
34
|
+
if edge.kind == EdgeKind.CONTAINS:
|
|
35
|
+
self.parent[edge.target] = edge.source
|
|
36
|
+
self.undirected[edge.source].append(edge.target)
|
|
37
|
+
self.undirected[edge.target].append(edge.source)
|
|
38
|
+
|
|
39
|
+
def module_of(self, chunk_id: str) -> str | None:
|
|
40
|
+
module_id = file_of(chunk_id) + MODULE_SUFFIX
|
|
41
|
+
return module_id if module_id in self.chunk_ids else None
|
|
42
|
+
|
|
43
|
+
def closure(self, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
|
|
44
|
+
"""Start set + outgoing CALLS/INHERITS within max_hops, + parent classes, + module chunks."""
|
|
45
|
+
result = {cid for cid in chunk_ids if cid in self.chunk_ids}
|
|
46
|
+
frontier = sorted(result)
|
|
47
|
+
for _ in range(max_hops):
|
|
48
|
+
next_frontier = []
|
|
49
|
+
for cid in frontier:
|
|
50
|
+
for target in self.depends_on[cid]:
|
|
51
|
+
if target not in result:
|
|
52
|
+
result.add(target)
|
|
53
|
+
next_frontier.append(target)
|
|
54
|
+
frontier = next_frontier
|
|
55
|
+
for cid in sorted(result):
|
|
56
|
+
if cid in self.parent:
|
|
57
|
+
result.add(self.parent[cid])
|
|
58
|
+
for cid in sorted(result):
|
|
59
|
+
module_id = self.module_of(cid)
|
|
60
|
+
if module_id is not None:
|
|
61
|
+
result.add(module_id)
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
def distances(
|
|
65
|
+
self, sources: list[str] | set[str], max_hops: int | None = None
|
|
66
|
+
) -> dict[str, int]:
|
|
67
|
+
"""Undirected multi-source BFS: chunk id -> hops to the nearest source."""
|
|
68
|
+
dist: dict[str, int] = {}
|
|
69
|
+
queue: deque[str] = deque()
|
|
70
|
+
for cid in sorted(sources):
|
|
71
|
+
if cid in self.chunk_ids and cid not in dist:
|
|
72
|
+
dist[cid] = 0
|
|
73
|
+
queue.append(cid)
|
|
74
|
+
while queue:
|
|
75
|
+
cid = queue.popleft()
|
|
76
|
+
if max_hops is not None and dist[cid] >= max_hops:
|
|
77
|
+
continue
|
|
78
|
+
for neighbour in self.undirected[cid]:
|
|
79
|
+
if neighbour not in dist:
|
|
80
|
+
dist[neighbour] = dist[cid] + 1
|
|
81
|
+
queue.append(neighbour)
|
|
82
|
+
return dist
|
|
83
|
+
|
|
84
|
+
def neighbours(self, chunk_id: str) -> list[str]:
|
|
85
|
+
"""Direct neighbours along any edge, either direction."""
|
|
86
|
+
return self.undirected.get(chunk_id, [])
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def dependency_closure(graph: CodeGraph, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
|
|
90
|
+
return GraphIndex(graph).closure(chunk_ids, max_hops)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def neighbours_within(graph: CodeGraph, chunk_id: str, k: int) -> dict[str, int]:
|
|
94
|
+
return GraphIndex(graph).distances([chunk_id], max_hops=k)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def multi_source_distances(graph: CodeGraph, sources: list[str] | set[str]) -> dict[str, int]:
|
|
98
|
+
return GraphIndex(graph).distances(sources)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def to_networkx(graph: CodeGraph) -> nx.MultiDiGraph:
|
|
102
|
+
g = nx.MultiDiGraph()
|
|
103
|
+
g.add_nodes_from(graph.chunk_ids)
|
|
104
|
+
for edge in graph.edges:
|
|
105
|
+
g.add_edge(edge.source, edge.target, kind=edge.kind.value, resolved=edge.resolved)
|
|
106
|
+
return g
|
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""LanguageParser interface (DECISIONS #6).
|
|
2
|
+
|
|
3
|
+
Python is the only language in Iteration 1, but parse_repo() talks to parsers
|
|
4
|
+
only through this Protocol so another language can be added later without
|
|
5
|
+
touching the graph or trimming code.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
from tokencut.contracts import Chunk
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class LanguageParser(Protocol):
|
|
14
|
+
file_suffixes: tuple[str, ...]
|
|
15
|
+
|
|
16
|
+
def parse_file(self, rel_path: str, source: bytes) -> list[Chunk]:
|
|
17
|
+
"""Return the chunks of one file. Must never raise on bad source code."""
|
|
18
|
+
...
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Member A — Repository ingestion and chunking."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from tokencut.contracts import Chunk
|
|
7
|
+
from tokencut.ingestion.python_parser import PythonParser
|
|
8
|
+
from tokencut.ingestion.walker import list_python_files
|
|
9
|
+
|
|
10
|
+
log = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def parse_repo(repo_path: str | Path) -> list[Chunk]:
|
|
14
|
+
"""Walk a repository and return one Chunk per module / class / function / method.
|
|
15
|
+
|
|
16
|
+
Must:
|
|
17
|
+
- skip ignored, binary and very large files
|
|
18
|
+
- never crash on a file with a syntax error (keep what tree-sitter can parse)
|
|
19
|
+
- return chunks in a deterministic order (file path, then start line)
|
|
20
|
+
"""
|
|
21
|
+
repo = Path(repo_path)
|
|
22
|
+
if not repo.is_dir():
|
|
23
|
+
raise FileNotFoundError(f"repository folder not found: {repo}")
|
|
24
|
+
|
|
25
|
+
parser = PythonParser()
|
|
26
|
+
chunks: list[Chunk] = []
|
|
27
|
+
for rel_path in list_python_files(repo):
|
|
28
|
+
try:
|
|
29
|
+
source = (repo / rel_path).read_bytes()
|
|
30
|
+
except OSError as exc:
|
|
31
|
+
log.warning("cannot read %s: %s", rel_path, exc)
|
|
32
|
+
continue
|
|
33
|
+
chunks.extend(parser.parse_file(rel_path, source))
|
|
34
|
+
|
|
35
|
+
chunks.sort(key=lambda c: (c.file_path, c.start_line, c.id))
|
|
36
|
+
return chunks
|