tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,307 @@
1
+ """Member B — Code-graph construction."""
2
+
3
+ import builtins
4
+
5
+ from tree_sitter import Node
6
+
7
+ from tokencut.contracts import Chunk, ChunkKind, CodeGraph, Edge, EdgeKind
8
+ from tokencut.graph.modules import ModuleMap, node_text, parse_code, read_imports
9
+
10
+ BUILTIN_NAMES = frozenset(dir(builtins))
11
+
12
+ # One entry of a file's name table: ("module", dotted name) or ("chunk", chunk id).
13
+ Binding = tuple[str, str]
14
+
15
+
16
+ def build_graph(chunks: list[Chunk]) -> CodeGraph:
17
+ """Build contains / imports / calls / inherits edges between chunks.
18
+
19
+ Must:
20
+ - never point an edge at a chunk id that does not exist
21
+ - mark heuristic call/import guesses with resolved=False
22
+ - handle circular imports without infinite loops
23
+ """
24
+ return _GraphBuilder(chunks).build()
25
+
26
+
27
+ def dependency_closure(graph: CodeGraph, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
28
+ """Return chunk_ids plus everything they depend on within max_hops."""
29
+ from tokencut.graph.queries import GraphIndex # local import: queries imports nothing from here
30
+
31
+ return GraphIndex(graph).closure(chunk_ids, max_hops)
32
+
33
+
34
+ class _GraphBuilder:
35
+ def __init__(self, chunks: list[Chunk]):
36
+ self.chunks = sorted(chunks, key=lambda c: (c.file_path, c.start_line, c.id))
37
+ self.by_id = {c.id: c for c in self.chunks}
38
+ self.modules = ModuleMap.build(self.chunks)
39
+ self.edges: dict[tuple[str, str, EdgeKind], bool] = {} # (source, target, kind) -> resolved
40
+ self.name_tables: dict[str, dict[str, Binding]] = {} # file path -> name table
41
+
42
+ # (class id, method name) -> method id; first definition wins (property getter before setter)
43
+ self.methods: dict[tuple[str, str], str] = {}
44
+ # last name -> chunk ids, for DECISIONS #7 rule 4
45
+ self.by_last_name: dict[str, list[str]] = {}
46
+ self.classes_by_name: dict[str, list[str]] = {}
47
+ for c in self.chunks:
48
+ if c.kind == ChunkKind.METHOD and c.parent_id:
49
+ self.methods.setdefault((c.parent_id, c.qualified_name.split(".")[-1]), c.id)
50
+ if c.kind != ChunkKind.MODULE:
51
+ last = c.qualified_name.split(".")[-1]
52
+ self.by_last_name.setdefault(last, []).append(c.id)
53
+ if c.kind == ChunkKind.CLASS:
54
+ self.classes_by_name.setdefault(last, []).append(c.id)
55
+ self.bases: dict[str, list[str]] = {} # class id -> resolved base class ids
56
+
57
+ # ---------- edge helpers ----------
58
+
59
+ def _add(self, source: str, target: str | None, kind: EdgeKind, resolved: bool = True) -> None:
60
+ if (
61
+ target is None
62
+ or source == target
63
+ or target not in self.by_id
64
+ or source not in self.by_id
65
+ ):
66
+ return
67
+ key = (source, target, kind)
68
+ self.edges[key] = self.edges.get(key, False) or resolved
69
+
70
+ def build(self) -> CodeGraph:
71
+ for c in self.chunks:
72
+ if c.kind == ChunkKind.MODULE:
73
+ self._imports(c)
74
+ for c in self.chunks:
75
+ if c.kind == ChunkKind.METHOD and c.parent_id:
76
+ self._add(c.parent_id, c.id, EdgeKind.CONTAINS)
77
+ elif c.kind == ChunkKind.CLASS:
78
+ self._inherits(c)
79
+ for c in self.chunks: # after INHERITS, because self.foo() may resolve into a base class
80
+ if c.kind in (ChunkKind.FUNCTION, ChunkKind.METHOD):
81
+ self._calls(c)
82
+
83
+ edges = [
84
+ Edge(source=s, target=t, kind=k, resolved=r)
85
+ for (s, t, k), r in sorted(
86
+ self.edges.items(), key=lambda kv: (kv[0][0], kv[0][1], kv[0][2].value)
87
+ )
88
+ ]
89
+ return CodeGraph(chunk_ids=[c.id for c in self.chunks], edges=edges)
90
+
91
+ # ---------- IMPORTS (plan B4) ----------
92
+
93
+ def _imports(self, module: Chunk) -> None:
94
+ table: dict[str, Binding] = {}
95
+ self.name_tables[module.file_path] = table
96
+ if not module.code.strip():
97
+ return
98
+ mm = self.modules
99
+ package = mm.package_of(module.file_path)
100
+ for imp in read_imports(parse_code(module.code), package):
101
+ if not imp.module:
102
+ continue
103
+ if imp.name is None and not imp.wildcard:
104
+ # import a.b / import a.b as z
105
+ if mm.is_module(imp.module):
106
+ self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
107
+ if imp.alias:
108
+ if mm.is_module(imp.module):
109
+ table[imp.alias] = ("module", imp.module)
110
+ else:
111
+ first = imp.module.split(".")[0]
112
+ if mm.is_module(first):
113
+ table[first] = ("module", first)
114
+ elif imp.wildcard:
115
+ if mm.is_module(imp.module):
116
+ self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
117
+ else:
118
+ # from a.b import y [as z]
119
+ local = imp.alias or imp.name
120
+ submodule = f"{imp.module}.{imp.name}"
121
+ if mm.is_module(submodule):
122
+ self._add(module.id, mm.module_chunk[submodule], EdgeKind.IMPORTS)
123
+ table[local] = ("module", submodule)
124
+ elif mm.is_module(imp.module):
125
+ self._add(module.id, mm.module_chunk[imp.module], EdgeKind.IMPORTS)
126
+ target = mm.top_names[imp.module].get(imp.name)
127
+ if target is not None:
128
+ table[local] = ("chunk", target)
129
+ # external module: ignored
130
+
131
+ # ---------- name resolution (DECISIONS #7) ----------
132
+
133
+ def _same_file_name(self, file_path: str, name: str) -> str | None:
134
+ module = self.modules.file_module.get(file_path)
135
+ if module is None:
136
+ return None
137
+ return self.modules.top_names[module].get(name)
138
+
139
+ def _unique(self, name: str, candidates: dict[str, list[str]]) -> str | None:
140
+ ids = candidates.get(name, [])
141
+ return ids[0] if len(ids) == 1 else None
142
+
143
+ def _resolve_name(
144
+ self, file_path: str, name: str, classes_only: bool = False
145
+ ) -> tuple[str | None, bool]:
146
+ """Resolve a bare identifier. Returns (chunk id or None, resolved flag)."""
147
+ binding = self.name_tables.get(file_path, {}).get(name)
148
+ if binding is not None: # rule 1: imported name
149
+ if binding[0] == "chunk" and self._kind_ok(binding[1], classes_only):
150
+ return binding[1], True
151
+ return None, True # a module, or the wrong kind: no edge, and no guessing
152
+ local = self._same_file_name(file_path, name) # rule 3: same file
153
+ if local is not None and self._kind_ok(local, classes_only):
154
+ return local, True
155
+ if name in BUILTIN_NAMES:
156
+ return None, True
157
+ return self._rule4(name, classes_only)
158
+
159
+ def _rule4(self, name: str, classes_only: bool = False) -> tuple[str | None, bool]:
160
+ # rule 4: exactly one definition with this name in the whole repo -> guess, resolved=False
161
+ target = self._unique(name, self.classes_by_name if classes_only else self.by_last_name)
162
+ return target, False
163
+
164
+ def _kind_ok(self, chunk_id: str, classes_only: bool) -> bool:
165
+ return not classes_only or self.by_id[chunk_id].kind == ChunkKind.CLASS
166
+
167
+ def _module_of_expr(self, file_path: str, node: Node) -> str | None:
168
+ """If node is `alias` or `alias.sub` naming an imported repo module, return its dotted name."""
169
+ if node.type == "identifier":
170
+ binding = self.name_tables.get(file_path, {}).get(node_text(node))
171
+ if binding is not None and binding[0] == "module":
172
+ return binding[1]
173
+ return None
174
+ if node.type == "attribute":
175
+ obj = node.child_by_field_name("object")
176
+ attr = node.child_by_field_name("attribute")
177
+ if obj is None or attr is None:
178
+ return None
179
+ parent = self._module_of_expr(file_path, obj)
180
+ if parent is not None and self.modules.is_module(f"{parent}.{node_text(attr)}"):
181
+ return f"{parent}.{node_text(attr)}"
182
+ return None
183
+
184
+ def _method_in_class_or_bases(self, class_id: str, name: str) -> str | None:
185
+ target = self.methods.get((class_id, name))
186
+ if target is not None:
187
+ return target
188
+ for base in self.bases.get(class_id, []): # one level of base classes
189
+ target = self.methods.get((base, name))
190
+ if target is not None:
191
+ return target
192
+ return None
193
+
194
+ def _class_binding(self, file_path: str, name: str) -> str | None:
195
+ """Chunk id if `name` is a class visible in this file (imported or defined here)."""
196
+ binding = self.name_tables.get(file_path, {}).get(name)
197
+ if binding is not None:
198
+ target = binding[1] if binding[0] == "chunk" else None
199
+ else:
200
+ target = self._same_file_name(file_path, name)
201
+ if target is not None and self.by_id[target].kind == ChunkKind.CLASS:
202
+ return target
203
+ return None
204
+
205
+ # ---------- INHERITS (plan B3) ----------
206
+
207
+ def _inherits(self, cls: Chunk) -> None:
208
+ # The class signature holds the bases; give it a body so tree-sitter sees a full class.
209
+ root = parse_code(cls.signature + "\n pass\n")
210
+ definition = next((n for n in _walk(root) if n.type == "class_definition"), None)
211
+ if definition is None:
212
+ return
213
+ superclasses = definition.child_by_field_name("superclasses")
214
+ if superclasses is None:
215
+ return
216
+ resolved_bases = []
217
+ for base in superclasses.named_children:
218
+ if base.type == "keyword_argument": # metaclass=...
219
+ continue
220
+ if base.type == "subscript": # Base[T]
221
+ base = base.child_by_field_name("value") or base
222
+ target, resolved = self._resolve_base(cls.file_path, base)
223
+ if target is not None:
224
+ self._add(cls.id, target, EdgeKind.INHERITS, resolved)
225
+ resolved_bases.append(target)
226
+ self.bases[cls.id] = resolved_bases
227
+
228
+ def _resolve_base(self, file_path: str, base: Node) -> tuple[str | None, bool]:
229
+ if base.type == "identifier":
230
+ return self._resolve_name(file_path, node_text(base), classes_only=True)
231
+ if base.type == "attribute":
232
+ obj = base.child_by_field_name("object")
233
+ name = node_text(base.child_by_field_name("attribute"))
234
+ module = self._module_of_expr(file_path, obj) if obj is not None else None
235
+ if module is not None:
236
+ target = self.modules.top_names[module].get(name)
237
+ if target is not None and self._kind_ok(target, True):
238
+ return target, True
239
+ return self._rule4(name, classes_only=True)
240
+ return None, True
241
+
242
+ # ---------- CALLS (plan B5) ----------
243
+
244
+ def _calls(self, func: Chunk) -> None:
245
+ root = parse_code(func.code)
246
+ for node in _walk(root):
247
+ if node.type != "call":
248
+ continue
249
+ callee = node.child_by_field_name("function")
250
+ if callee is None:
251
+ continue
252
+ target, resolved = self._resolve_callee(func, callee)
253
+ self._add(func.id, target, EdgeKind.CALLS, resolved)
254
+
255
+ def _resolve_callee(self, func: Chunk, callee: Node) -> tuple[str | None, bool]:
256
+ file_path = func.file_path
257
+ if callee.type == "identifier":
258
+ return self._resolve_name(file_path, node_text(callee))
259
+ if callee.type != "attribute":
260
+ return None, True # e.g. f()() or x[0]()
261
+ obj = callee.child_by_field_name("object")
262
+ attr = callee.child_by_field_name("attribute")
263
+ if obj is None or attr is None:
264
+ return None, True
265
+ name = node_text(attr)
266
+ in_method = func.kind == ChunkKind.METHOD and func.parent_id is not None
267
+
268
+ # rule 2: self.foo() / cls.foo() inside a method -> same class, then its base classes
269
+ if in_method and obj.type == "identifier" and node_text(obj) in ("self", "cls"):
270
+ target = self._method_in_class_or_bases(func.parent_id, name)
271
+ if target is not None:
272
+ return target, True
273
+ # super().foo() -> base classes only
274
+ elif in_method and obj.type == "call" and _is_super_call(obj):
275
+ for base in self.bases.get(func.parent_id, []):
276
+ target = self.methods.get((base, name))
277
+ if target is not None:
278
+ return target, True
279
+ else:
280
+ # rule 1: module_alias.foo()
281
+ module = self._module_of_expr(file_path, obj)
282
+ if module is not None:
283
+ target = self.modules.top_names[module].get(name)
284
+ if target is not None:
285
+ return target, True
286
+ # rule 1 / 3: SomeClass.method() where the class is imported or defined in this file
287
+ elif obj.type == "identifier":
288
+ class_id = self._class_binding(file_path, node_text(obj))
289
+ if class_id is not None:
290
+ target = self._method_in_class_or_bases(class_id, name)
291
+ if target is not None:
292
+ return target, True
293
+ return self._rule4(name)
294
+
295
+
296
+ def _is_super_call(node: Node) -> bool:
297
+ fn = node.child_by_field_name("function")
298
+ return fn is not None and fn.type == "identifier" and node_text(fn) == "super"
299
+
300
+
301
+ def _walk(root: Node):
302
+ """All nodes under root in source order (explicit stack: no recursion limit on deep code)."""
303
+ stack = [root]
304
+ while stack:
305
+ node = stack.pop()
306
+ yield node
307
+ stack.extend(reversed(node.children))
@@ -0,0 +1,149 @@
1
+ """File path <-> dotted module name, and reading import statements (IMPLEMENTATION_PLAN B2, B4)."""
2
+
3
+ import textwrap
4
+ from dataclasses import dataclass, field
5
+
6
+ import tree_sitter_python as tspython
7
+ from tree_sitter import Language, Node, Parser
8
+
9
+ from tokencut.contracts import Chunk, ChunkKind
10
+
11
+ _PARSER = Parser(Language(tspython.language()))
12
+
13
+ # Source-layout folders that are not part of the import name (pytest uses src/_pytest/...).
14
+ LAYOUT_PREFIXES = ("src", "lib")
15
+
16
+
17
+ def parse_code(code: str) -> Node:
18
+ """Parse a chunk's code with tree-sitter. Methods are indented, so dedent first."""
19
+ return _PARSER.parse(textwrap.dedent(code).encode("utf-8")).root_node
20
+
21
+
22
+ def node_text(node: Node) -> str:
23
+ return node.text.decode("utf-8", errors="replace")
24
+
25
+
26
+ def module_name_for(file_path: str) -> str:
27
+ """'shop/cart.py' -> 'shop.cart'; 'shop/payments/__init__.py' -> 'shop.payments'."""
28
+ parts = (
29
+ file_path[: -len(".py")].split("/") if file_path.endswith(".py") else file_path.split("/")
30
+ )
31
+ if parts and parts[-1] == "__init__" and len(parts) > 1:
32
+ parts = parts[:-1]
33
+ return ".".join(parts)
34
+
35
+
36
+ def is_package_file(file_path: str) -> bool:
37
+ return file_path.endswith("/__init__.py") or file_path == "__init__.py"
38
+
39
+
40
+ @dataclass
41
+ class ModuleMap:
42
+ """Lookup tables over all module chunks of the repo."""
43
+
44
+ module_chunk: dict[str, str] = field(default_factory=dict) # module name -> module chunk id
45
+ file_module: dict[str, str] = field(default_factory=dict) # file path -> primary module name
46
+ top_names: dict[str, dict[str, str]] = field(
47
+ default_factory=dict
48
+ ) # module -> {name -> chunk id}
49
+
50
+ @classmethod
51
+ def build(cls, chunks: list[Chunk]) -> "ModuleMap":
52
+ mm = cls()
53
+ files = sorted({c.file_path for c in chunks if c.kind == ChunkKind.MODULE})
54
+ names_by_file: dict[str, dict[str, str]] = {f: {} for f in files}
55
+ for chunk in chunks: # chunks are in line order, so a later definition wins (as in Python)
56
+ if (
57
+ chunk.kind in (ChunkKind.FUNCTION, ChunkKind.CLASS)
58
+ and chunk.file_path in names_by_file
59
+ ):
60
+ names_by_file[chunk.file_path][chunk.qualified_name] = chunk.id
61
+
62
+ # Primary names first, so a 'src/'-less alias never hides a real module of that name.
63
+ for file_path in files:
64
+ name = module_name_for(file_path)
65
+ mm.file_module[file_path] = name
66
+ mm.module_chunk[name] = f"{file_path}::<module>"
67
+ mm.top_names[name] = names_by_file[file_path]
68
+ for file_path in files:
69
+ name = mm.file_module[file_path]
70
+ first, _, rest = name.partition(".")
71
+ if first in LAYOUT_PREFIXES and rest and rest not in mm.module_chunk:
72
+ mm.module_chunk[rest] = mm.module_chunk[name]
73
+ mm.top_names[rest] = mm.top_names[name]
74
+ return mm
75
+
76
+ def is_module(self, name: str) -> bool:
77
+ return name in self.module_chunk
78
+
79
+ def package_of(self, file_path: str) -> str:
80
+ """The package that relative imports in this file are resolved against."""
81
+ name = self.file_module[file_path]
82
+ if is_package_file(file_path):
83
+ return name
84
+ return name.rpartition(".")[0]
85
+
86
+
87
+ @dataclass
88
+ class ImportedName:
89
+ """One name bound by an import statement."""
90
+
91
+ module: str # absolute dotted module the statement refers to ('' if unresolvable)
92
+ name: str | None # imported name for 'from m import name'; None for 'import m'
93
+ alias: str | None # local binding if 'as' was used
94
+ wildcard: bool = False
95
+
96
+
97
+ def read_imports(module_root: Node, package: str) -> list[ImportedName]:
98
+ """Collect import statements anywhere in a module chunk (e.g. also inside try/if blocks)."""
99
+ found: list[ImportedName] = []
100
+ stack = [module_root]
101
+ while stack:
102
+ node = stack.pop()
103
+ if node.type == "import_statement":
104
+ for child in node.children_by_field_name("name"):
105
+ dotted, alias = _name_and_alias(child)
106
+ found.append(ImportedName(module=dotted, name=None, alias=alias))
107
+ elif node.type == "import_from_statement":
108
+ module_node = node.child_by_field_name("module_name")
109
+ base = _resolve_from_module(module_node, package) if module_node is not None else ""
110
+ names = node.children_by_field_name("name")
111
+ if any(c.type == "wildcard_import" for c in node.children):
112
+ found.append(ImportedName(module=base, name=None, alias=None, wildcard=True))
113
+ for child in names:
114
+ dotted, alias = _name_and_alias(child)
115
+ found.append(ImportedName(module=base, name=dotted, alias=alias))
116
+ else:
117
+ stack.extend(reversed(node.children))
118
+ return found
119
+
120
+
121
+ def _name_and_alias(node: Node) -> tuple[str, str | None]:
122
+ if node.type == "aliased_import":
123
+ name_node = node.child_by_field_name("name")
124
+ alias_node = node.child_by_field_name("alias")
125
+ return (
126
+ node_text(name_node) if name_node else "",
127
+ node_text(alias_node) if alias_node else None,
128
+ )
129
+ return node_text(node), None
130
+
131
+
132
+ def _resolve_from_module(module_node: Node, package: str) -> str:
133
+ """'from ..a.b import x' inside package 'p.q.r' -> 'p.q.a.b'. '' if it escapes the repo."""
134
+ if module_node.type != "relative_import":
135
+ return node_text(module_node)
136
+ level = 0
137
+ rest = ""
138
+ for child in module_node.children:
139
+ if child.type == "import_prefix":
140
+ level = node_text(child).count(".")
141
+ elif child.type == "dotted_name":
142
+ rest = node_text(child)
143
+ parts = package.split(".") if package else []
144
+ if level - 1 > len(parts):
145
+ return ""
146
+ base_parts = parts[: len(parts) - (level - 1)]
147
+ if rest:
148
+ base_parts = base_parts + rest.split(".")
149
+ return ".".join(base_parts)
@@ -0,0 +1,106 @@
1
+ """Read-only questions about a CodeGraph (IMPLEMENTATION_PLAN B6).
2
+
3
+ GraphIndex pre-computes adjacency lists once; the module-level functions are
4
+ convenience wrappers for one-off calls. Results are sets/dicts: callers must
5
+ sort before producing output (DECISIONS #8).
6
+ """
7
+
8
+ from collections import deque
9
+
10
+ import networkx as nx
11
+
12
+ from tokencut.contracts import CodeGraph, EdgeKind
13
+
14
+ MODULE_SUFFIX = "::<module>"
15
+ _DEPENDENCY_KINDS = (EdgeKind.CALLS, EdgeKind.INHERITS)
16
+
17
+
18
+ def file_of(chunk_id: str) -> str:
19
+ """'shop/cart.py::Cart.total' -> 'shop/cart.py' (file paths never contain '::')."""
20
+ return chunk_id.split("::", 1)[0]
21
+
22
+
23
+ class GraphIndex:
24
+ def __init__(self, graph: CodeGraph):
25
+ self.chunk_ids = set(graph.chunk_ids)
26
+ self.depends_on: dict[str, list[str]] = {cid: [] for cid in graph.chunk_ids}
27
+ self.undirected: dict[str, list[str]] = {cid: [] for cid in graph.chunk_ids}
28
+ self.parent: dict[str, str] = {} # method id -> class id (from CONTAINS edges)
29
+ for edge in graph.edges: # graph.edges is sorted, so neighbour lists are deterministic
30
+ if edge.source not in self.chunk_ids or edge.target not in self.chunk_ids:
31
+ continue
32
+ if edge.kind in _DEPENDENCY_KINDS:
33
+ self.depends_on[edge.source].append(edge.target)
34
+ if edge.kind == EdgeKind.CONTAINS:
35
+ self.parent[edge.target] = edge.source
36
+ self.undirected[edge.source].append(edge.target)
37
+ self.undirected[edge.target].append(edge.source)
38
+
39
+ def module_of(self, chunk_id: str) -> str | None:
40
+ module_id = file_of(chunk_id) + MODULE_SUFFIX
41
+ return module_id if module_id in self.chunk_ids else None
42
+
43
+ def closure(self, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
44
+ """Start set + outgoing CALLS/INHERITS within max_hops, + parent classes, + module chunks."""
45
+ result = {cid for cid in chunk_ids if cid in self.chunk_ids}
46
+ frontier = sorted(result)
47
+ for _ in range(max_hops):
48
+ next_frontier = []
49
+ for cid in frontier:
50
+ for target in self.depends_on[cid]:
51
+ if target not in result:
52
+ result.add(target)
53
+ next_frontier.append(target)
54
+ frontier = next_frontier
55
+ for cid in sorted(result):
56
+ if cid in self.parent:
57
+ result.add(self.parent[cid])
58
+ for cid in sorted(result):
59
+ module_id = self.module_of(cid)
60
+ if module_id is not None:
61
+ result.add(module_id)
62
+ return result
63
+
64
+ def distances(
65
+ self, sources: list[str] | set[str], max_hops: int | None = None
66
+ ) -> dict[str, int]:
67
+ """Undirected multi-source BFS: chunk id -> hops to the nearest source."""
68
+ dist: dict[str, int] = {}
69
+ queue: deque[str] = deque()
70
+ for cid in sorted(sources):
71
+ if cid in self.chunk_ids and cid not in dist:
72
+ dist[cid] = 0
73
+ queue.append(cid)
74
+ while queue:
75
+ cid = queue.popleft()
76
+ if max_hops is not None and dist[cid] >= max_hops:
77
+ continue
78
+ for neighbour in self.undirected[cid]:
79
+ if neighbour not in dist:
80
+ dist[neighbour] = dist[cid] + 1
81
+ queue.append(neighbour)
82
+ return dist
83
+
84
+ def neighbours(self, chunk_id: str) -> list[str]:
85
+ """Direct neighbours along any edge, either direction."""
86
+ return self.undirected.get(chunk_id, [])
87
+
88
+
89
+ def dependency_closure(graph: CodeGraph, chunk_ids: set[str], max_hops: int = 1) -> set[str]:
90
+ return GraphIndex(graph).closure(chunk_ids, max_hops)
91
+
92
+
93
+ def neighbours_within(graph: CodeGraph, chunk_id: str, k: int) -> dict[str, int]:
94
+ return GraphIndex(graph).distances([chunk_id], max_hops=k)
95
+
96
+
97
+ def multi_source_distances(graph: CodeGraph, sources: list[str] | set[str]) -> dict[str, int]:
98
+ return GraphIndex(graph).distances(sources)
99
+
100
+
101
+ def to_networkx(graph: CodeGraph) -> nx.MultiDiGraph:
102
+ g = nx.MultiDiGraph()
103
+ g.add_nodes_from(graph.chunk_ids)
104
+ for edge in graph.edges:
105
+ g.add_edge(edge.source, edge.target, kind=edge.kind.value, resolved=edge.resolved)
106
+ return g
File without changes
@@ -0,0 +1,18 @@
1
+ """LanguageParser interface (DECISIONS #6).
2
+
3
+ Python is the only language in Iteration 1, but parse_repo() talks to parsers
4
+ only through this Protocol so another language can be added later without
5
+ touching the graph or trimming code.
6
+ """
7
+
8
+ from typing import Protocol
9
+
10
+ from tokencut.contracts import Chunk
11
+
12
+
13
+ class LanguageParser(Protocol):
14
+ file_suffixes: tuple[str, ...]
15
+
16
+ def parse_file(self, rel_path: str, source: bytes) -> list[Chunk]:
17
+ """Return the chunks of one file. Must never raise on bad source code."""
18
+ ...
@@ -0,0 +1,36 @@
1
+ """Member A — Repository ingestion and chunking."""
2
+
3
+ import logging
4
+ from pathlib import Path
5
+
6
+ from tokencut.contracts import Chunk
7
+ from tokencut.ingestion.python_parser import PythonParser
8
+ from tokencut.ingestion.walker import list_python_files
9
+
10
+ log = logging.getLogger(__name__)
11
+
12
+
13
+ def parse_repo(repo_path: str | Path) -> list[Chunk]:
14
+ """Walk a repository and return one Chunk per module / class / function / method.
15
+
16
+ Must:
17
+ - skip ignored, binary and very large files
18
+ - never crash on a file with a syntax error (keep what tree-sitter can parse)
19
+ - return chunks in a deterministic order (file path, then start line)
20
+ """
21
+ repo = Path(repo_path)
22
+ if not repo.is_dir():
23
+ raise FileNotFoundError(f"repository folder not found: {repo}")
24
+
25
+ parser = PythonParser()
26
+ chunks: list[Chunk] = []
27
+ for rel_path in list_python_files(repo):
28
+ try:
29
+ source = (repo / rel_path).read_bytes()
30
+ except OSError as exc:
31
+ log.warning("cannot read %s: %s", rel_path, exc)
32
+ continue
33
+ chunks.extend(parser.parse_file(rel_path, source))
34
+
35
+ chunks.sort(key=lambda c: (c.file_path, c.start_line, c.id))
36
+ return chunks