codemble 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codemble/__init__.py +4 -0
- codemble/adapters/__init__.py +38 -0
- codemble/adapters/base.py +199 -0
- codemble/adapters/discovery.py +190 -0
- codemble/adapters/project.py +206 -0
- codemble/adapters/python_ast.py +766 -0
- codemble/adapters/typescript_tree_sitter.py +1263 -0
- codemble/checks/__init__.py +19 -0
- codemble/checks/service.py +380 -0
- codemble/cli.py +161 -0
- codemble/graph/__init__.py +17 -0
- codemble/graph/finalize.py +91 -0
- codemble/graph/layout.py +132 -0
- codemble/lens/__init__.py +18 -0
- codemble/lens/javascript_typescript.py +82 -0
- codemble/lens/python.py +71 -0
- codemble/llm/__init__.py +6 -0
- codemble/llm/providers.py +137 -0
- codemble/llm/study.py +439 -0
- codemble/progress/__init__.py +9 -0
- codemble/progress/store.py +160 -0
- codemble/server/__init__.py +6 -0
- codemble/server/app.py +273 -0
- codemble/server/runtime.py +68 -0
- codemble/web_dist/assets/index-DOBVd_-M.css +1 -0
- codemble/web_dist/assets/index-cIG6GGIB.js +5238 -0
- codemble/web_dist/assets/jetbrains-mono-latin-400-normal-6-qcROiO.woff +0 -0
- codemble/web_dist/assets/jetbrains-mono-latin-400-normal-V6pRDFza.woff2 +0 -0
- codemble/web_dist/assets/jetbrains-mono-latin-500-normal-BWZEU5yA.woff2 +0 -0
- codemble/web_dist/assets/jetbrains-mono-latin-500-normal-CJOVTJB7.woff +0 -0
- codemble/web_dist/assets/shippori-mincho-latin-500-normal-C-QwvIb3.woff +0 -0
- codemble/web_dist/assets/shippori-mincho-latin-500-normal-XI1O8euf.woff2 +0 -0
- codemble/web_dist/assets/shippori-mincho-latin-700-normal-CkoCYOiI.woff +0 -0
- codemble/web_dist/assets/shippori-mincho-latin-700-normal-DHcmzUO5.woff2 +0 -0
- codemble/web_dist/assets/zen-kaku-gothic-new-latin-400-normal-BEdayliK.woff2 +0 -0
- codemble/web_dist/assets/zen-kaku-gothic-new-latin-400-normal-CPSmNJAU.woff +0 -0
- codemble/web_dist/index.html +18 -0
- codemble-0.3.0.dist-info/METADATA +417 -0
- codemble-0.3.0.dist-info/RECORD +43 -0
- codemble-0.3.0.dist-info/WHEEL +4 -0
- codemble-0.3.0.dist-info/entry_points.txt +2 -0
- codemble-0.3.0.dist-info/licenses/LICENSE +202 -0
- codemble-0.3.0.dist-info/licenses/NOTICE +2 -0
|
@@ -0,0 +1,766 @@
|
|
|
1
|
+
"""Python's stdlib-AST implementation of the Codemble language seam."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import builtins
|
|
7
|
+
import hashlib
|
|
8
|
+
import tokenize
|
|
9
|
+
from collections import defaultdict
|
|
10
|
+
from dataclasses import dataclass, replace
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from codemble.adapters.base import AdapterParseError, ConceptAnnotation, Edge, Graph, Node
|
|
14
|
+
from codemble.adapters.discovery import SourceDiscoveryError, discover_source_files
|
|
15
|
+
from codemble.graph.finalize import GraphFinalizationError, finalize_graph
|
|
16
|
+
|
|
17
|
+
_APP_FACTORIES = {"FastAPI", "Flask", "Typer"}
|
|
18
|
+
_BUILTIN_NAMES = frozenset(dir(builtins))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PythonParseError(AdapterParseError):
|
|
22
|
+
"""The requested Python project cannot be discovered safely."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class _ParsedFile:
|
|
27
|
+
path: Path
|
|
28
|
+
relative_path: str
|
|
29
|
+
module: str
|
|
30
|
+
source: str
|
|
31
|
+
digest: str
|
|
32
|
+
tree: ast.Module | None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, slots=True)
|
|
36
|
+
class _Definition:
|
|
37
|
+
node_id: str
|
|
38
|
+
syntax: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef
|
|
39
|
+
parent_id: str
|
|
40
|
+
enclosing_class_id: str | None
|
|
41
|
+
function_ancestors: tuple[str, ...]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class _ImportBinding:
|
|
46
|
+
local_name: str
|
|
47
|
+
target: str
|
|
48
|
+
external: bool
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _DefinitionCollector(ast.NodeVisitor):
|
|
52
|
+
"""Collect definitions while retaining their lexical ownership."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, parsed: _ParsedFile) -> None:
|
|
55
|
+
self.parsed = parsed
|
|
56
|
+
self.nodes: list[Node] = []
|
|
57
|
+
self.definitions: list[_Definition] = []
|
|
58
|
+
self._qualname: list[str] = []
|
|
59
|
+
self._scope_ids: list[str] = [parsed.module]
|
|
60
|
+
self._class_ids: list[str] = []
|
|
61
|
+
self._function_ids: list[str] = []
|
|
62
|
+
|
|
63
|
+
def visit_ClassDef(self, syntax: ast.ClassDef) -> None:
|
|
64
|
+
node_id = self._node_id(syntax.name)
|
|
65
|
+
self.nodes.append(self._node(syntax, node_id, "class"))
|
|
66
|
+
self.definitions.append(
|
|
67
|
+
_Definition(
|
|
68
|
+
node_id=node_id,
|
|
69
|
+
syntax=syntax,
|
|
70
|
+
parent_id=self._scope_ids[-1],
|
|
71
|
+
enclosing_class_id=self._class_ids[-1] if self._class_ids else None,
|
|
72
|
+
function_ancestors=tuple(self._function_ids),
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
self._qualname.append(syntax.name)
|
|
76
|
+
self._scope_ids.append(node_id)
|
|
77
|
+
self._class_ids.append(node_id)
|
|
78
|
+
self.generic_visit(syntax)
|
|
79
|
+
self._class_ids.pop()
|
|
80
|
+
self._scope_ids.pop()
|
|
81
|
+
self._qualname.pop()
|
|
82
|
+
|
|
83
|
+
def visit_FunctionDef(self, syntax: ast.FunctionDef) -> None:
|
|
84
|
+
self._visit_function(syntax)
|
|
85
|
+
|
|
86
|
+
def visit_AsyncFunctionDef(self, syntax: ast.AsyncFunctionDef) -> None:
|
|
87
|
+
self._visit_function(syntax)
|
|
88
|
+
|
|
89
|
+
def _visit_function(self, syntax: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
|
|
90
|
+
node_id = self._node_id(syntax.name)
|
|
91
|
+
self.nodes.append(self._node(syntax, node_id, "function"))
|
|
92
|
+
self.definitions.append(
|
|
93
|
+
_Definition(
|
|
94
|
+
node_id=node_id,
|
|
95
|
+
syntax=syntax,
|
|
96
|
+
parent_id=self._scope_ids[-1],
|
|
97
|
+
enclosing_class_id=self._class_ids[-1] if self._class_ids else None,
|
|
98
|
+
function_ancestors=tuple(self._function_ids),
|
|
99
|
+
)
|
|
100
|
+
)
|
|
101
|
+
self._qualname.append(syntax.name)
|
|
102
|
+
self._scope_ids.append(node_id)
|
|
103
|
+
self._function_ids.append(node_id)
|
|
104
|
+
self.generic_visit(syntax)
|
|
105
|
+
self._function_ids.pop()
|
|
106
|
+
self._scope_ids.pop()
|
|
107
|
+
self._qualname.pop()
|
|
108
|
+
|
|
109
|
+
def _node_id(self, name: str) -> str:
|
|
110
|
+
return ".".join((self.parsed.module, *self._qualname, name))
|
|
111
|
+
|
|
112
|
+
def _node(
|
|
113
|
+
self,
|
|
114
|
+
syntax: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef,
|
|
115
|
+
node_id: str,
|
|
116
|
+
kind: str,
|
|
117
|
+
) -> Node:
|
|
118
|
+
end_lineno = syntax.end_lineno or syntax.lineno
|
|
119
|
+
return Node(
|
|
120
|
+
id=node_id,
|
|
121
|
+
kind=kind, # type: ignore[arg-type]
|
|
122
|
+
name=syntax.name,
|
|
123
|
+
language="python",
|
|
124
|
+
file=self.parsed.relative_path,
|
|
125
|
+
lineno=syntax.lineno,
|
|
126
|
+
end_lineno=end_lineno,
|
|
127
|
+
loc=end_lineno - syntax.lineno + 1,
|
|
128
|
+
region=_region_for(self.parsed.module),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _ScopeFacts(ast.NodeVisitor):
|
|
133
|
+
"""Collect facts owned by one lexical scope, excluding nested scopes."""
|
|
134
|
+
|
|
135
|
+
def __init__(self) -> None:
|
|
136
|
+
self.imports: list[ast.Import | ast.ImportFrom] = []
|
|
137
|
+
self.calls: list[ast.Call] = []
|
|
138
|
+
|
|
139
|
+
def visit_Import(self, syntax: ast.Import) -> None:
|
|
140
|
+
self.imports.append(syntax)
|
|
141
|
+
|
|
142
|
+
def visit_ImportFrom(self, syntax: ast.ImportFrom) -> None:
|
|
143
|
+
self.imports.append(syntax)
|
|
144
|
+
|
|
145
|
+
def visit_Call(self, syntax: ast.Call) -> None:
|
|
146
|
+
self.calls.append(syntax)
|
|
147
|
+
self.generic_visit(syntax)
|
|
148
|
+
|
|
149
|
+
def visit_ClassDef(self, syntax: ast.ClassDef) -> None:
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
def visit_FunctionDef(self, syntax: ast.FunctionDef) -> None:
|
|
153
|
+
return
|
|
154
|
+
|
|
155
|
+
def visit_AsyncFunctionDef(self, syntax: ast.AsyncFunctionDef) -> None:
|
|
156
|
+
return
|
|
157
|
+
|
|
158
|
+
def collect(self, statements: list[ast.stmt]) -> _ScopeFacts:
|
|
159
|
+
for statement in statements:
|
|
160
|
+
self.visit(statement)
|
|
161
|
+
return self
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class _ConceptCollector(ast.NodeVisitor):
|
|
165
|
+
"""Collect concepts inside one lexical owner without crossing into children."""
|
|
166
|
+
|
|
167
|
+
def __init__(self, node_id: str, language: str, source: str) -> None:
|
|
168
|
+
self.node_id = node_id
|
|
169
|
+
self.language = language
|
|
170
|
+
self.source_lines = source.splitlines()
|
|
171
|
+
self.annotations: set[ConceptAnnotation] = set()
|
|
172
|
+
|
|
173
|
+
def add(self, concept: str, syntax: ast.AST) -> None:
|
|
174
|
+
lineno = getattr(syntax, "lineno", None)
|
|
175
|
+
if not isinstance(lineno, int):
|
|
176
|
+
return
|
|
177
|
+
end_lineno = getattr(syntax, "end_lineno", lineno)
|
|
178
|
+
if not isinstance(end_lineno, int):
|
|
179
|
+
end_lineno = lineno
|
|
180
|
+
snippet = self.source_lines[lineno - 1].strip() if lineno <= len(self.source_lines) else ""
|
|
181
|
+
self.annotations.add(
|
|
182
|
+
ConceptAnnotation(
|
|
183
|
+
node_id=self.node_id,
|
|
184
|
+
language=self.language,
|
|
185
|
+
concept=concept,
|
|
186
|
+
lineno=lineno,
|
|
187
|
+
end_lineno=end_lineno,
|
|
188
|
+
snippet=snippet[:240],
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def visit_FunctionDef(self, syntax: ast.FunctionDef) -> None:
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
def visit_AsyncFunctionDef(self, syntax: ast.AsyncFunctionDef) -> None:
|
|
196
|
+
return
|
|
197
|
+
|
|
198
|
+
def visit_ClassDef(self, syntax: ast.ClassDef) -> None:
|
|
199
|
+
return
|
|
200
|
+
|
|
201
|
+
def visit_ListComp(self, syntax: ast.ListComp) -> None:
|
|
202
|
+
self.add("comprehension", syntax)
|
|
203
|
+
self.generic_visit(syntax)
|
|
204
|
+
|
|
205
|
+
def visit_SetComp(self, syntax: ast.SetComp) -> None:
|
|
206
|
+
self.add("comprehension", syntax)
|
|
207
|
+
self.generic_visit(syntax)
|
|
208
|
+
|
|
209
|
+
def visit_DictComp(self, syntax: ast.DictComp) -> None:
|
|
210
|
+
self.add("comprehension", syntax)
|
|
211
|
+
self.generic_visit(syntax)
|
|
212
|
+
|
|
213
|
+
def visit_GeneratorExp(self, syntax: ast.GeneratorExp) -> None:
|
|
214
|
+
self.add("generator", syntax)
|
|
215
|
+
self.generic_visit(syntax)
|
|
216
|
+
|
|
217
|
+
def visit_Yield(self, syntax: ast.Yield) -> None:
|
|
218
|
+
self.add("generator", syntax)
|
|
219
|
+
self.generic_visit(syntax)
|
|
220
|
+
|
|
221
|
+
def visit_YieldFrom(self, syntax: ast.YieldFrom) -> None:
|
|
222
|
+
self.add("generator", syntax)
|
|
223
|
+
self.generic_visit(syntax)
|
|
224
|
+
|
|
225
|
+
def visit_With(self, syntax: ast.With) -> None:
|
|
226
|
+
self.add("context-manager", syntax)
|
|
227
|
+
self.generic_visit(syntax)
|
|
228
|
+
|
|
229
|
+
def visit_AsyncWith(self, syntax: ast.AsyncWith) -> None:
|
|
230
|
+
self.add("context-manager", syntax)
|
|
231
|
+
self.add("async-await", syntax)
|
|
232
|
+
self.generic_visit(syntax)
|
|
233
|
+
|
|
234
|
+
def visit_Await(self, syntax: ast.Await) -> None:
|
|
235
|
+
self.add("async-await", syntax)
|
|
236
|
+
self.generic_visit(syntax)
|
|
237
|
+
|
|
238
|
+
def visit_AsyncFor(self, syntax: ast.AsyncFor) -> None:
|
|
239
|
+
self.add("async-await", syntax)
|
|
240
|
+
self.generic_visit(syntax)
|
|
241
|
+
|
|
242
|
+
def visit_Try(self, syntax: ast.Try) -> None:
|
|
243
|
+
self.add("exception-handling", syntax)
|
|
244
|
+
self.generic_visit(syntax)
|
|
245
|
+
|
|
246
|
+
def visit_TryStar(self, syntax: ast.TryStar) -> None:
|
|
247
|
+
self.add("exception-handling", syntax)
|
|
248
|
+
self.generic_visit(syntax)
|
|
249
|
+
|
|
250
|
+
def visit_Raise(self, syntax: ast.Raise) -> None:
|
|
251
|
+
self.add("exception-handling", syntax)
|
|
252
|
+
self.generic_visit(syntax)
|
|
253
|
+
|
|
254
|
+
def visit_AnnAssign(self, syntax: ast.AnnAssign) -> None:
|
|
255
|
+
self.add("type-hint", syntax.annotation)
|
|
256
|
+
self.generic_visit(syntax)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _concepts_for_target(
|
|
260
|
+
node: Node,
|
|
261
|
+
target: ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef,
|
|
262
|
+
source: str,
|
|
263
|
+
) -> list[ConceptAnnotation]:
|
|
264
|
+
collector = _ConceptCollector(node.id, node.language, source)
|
|
265
|
+
if isinstance(target, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
266
|
+
for decorator in target.decorator_list:
|
|
267
|
+
collector.add("decorator", decorator)
|
|
268
|
+
if isinstance(target, ast.AsyncFunctionDef):
|
|
269
|
+
collector.add("async-await", target)
|
|
270
|
+
if isinstance(target, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
271
|
+
if target.name.startswith("__") and target.name.endswith("__"):
|
|
272
|
+
collector.add("dunder-method", target)
|
|
273
|
+
arguments = [
|
|
274
|
+
*target.args.posonlyargs,
|
|
275
|
+
*target.args.args,
|
|
276
|
+
*target.args.kwonlyargs,
|
|
277
|
+
]
|
|
278
|
+
if target.args.vararg is not None:
|
|
279
|
+
arguments.append(target.args.vararg)
|
|
280
|
+
if target.args.kwarg is not None:
|
|
281
|
+
arguments.append(target.args.kwarg)
|
|
282
|
+
for argument in arguments:
|
|
283
|
+
if argument.annotation is not None:
|
|
284
|
+
collector.add("type-hint", argument.annotation)
|
|
285
|
+
if target.returns is not None:
|
|
286
|
+
collector.add("type-hint", target.returns)
|
|
287
|
+
for statement in target.body:
|
|
288
|
+
collector.visit(statement)
|
|
289
|
+
return sorted(
|
|
290
|
+
collector.annotations,
|
|
291
|
+
key=lambda item: (item.lineno, item.concept, item.end_lineno, item.snippet),
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class PythonAstAdapter:
|
|
296
|
+
"""Parse Python source into deterministic, render-ready graph data."""
|
|
297
|
+
|
|
298
|
+
language = "python"
|
|
299
|
+
file_extensions = frozenset({".py"})
|
|
300
|
+
ignored_directories = frozenset()
|
|
301
|
+
|
|
302
|
+
def discover(self, path: Path) -> tuple[Path, tuple[Path, ...]]:
|
|
303
|
+
"""Return the parser's exact project root and accepted Python files."""
|
|
304
|
+
|
|
305
|
+
return _discover_python_files(path.expanduser().resolve())
|
|
306
|
+
|
|
307
|
+
def parse(self, path: Path, *, entrypoint: str | None = None) -> Graph:
|
|
308
|
+
project_root, files = self.discover(path)
|
|
309
|
+
return self.parse_files(project_root, files, entrypoint=entrypoint)
|
|
310
|
+
|
|
311
|
+
def parse_files(
|
|
312
|
+
self,
|
|
313
|
+
project_root: Path,
|
|
314
|
+
files: tuple[Path, ...],
|
|
315
|
+
*,
|
|
316
|
+
entrypoint: str | None = None,
|
|
317
|
+
) -> Graph:
|
|
318
|
+
"""Parse Python files already owned by this adapter."""
|
|
319
|
+
|
|
320
|
+
parsed_files = tuple(_parse_file(file, project_root) for file in files)
|
|
321
|
+
|
|
322
|
+
nodes: list[Node] = []
|
|
323
|
+
definitions: list[_Definition] = []
|
|
324
|
+
definition_by_id: dict[str, _Definition] = {}
|
|
325
|
+
parsed_by_module = {parsed.module: parsed for parsed in parsed_files}
|
|
326
|
+
modules = set(parsed_by_module)
|
|
327
|
+
|
|
328
|
+
for parsed in parsed_files:
|
|
329
|
+
module_node = _module_node(parsed)
|
|
330
|
+
nodes.append(module_node)
|
|
331
|
+
if parsed.tree is None:
|
|
332
|
+
continue
|
|
333
|
+
collector = _DefinitionCollector(parsed)
|
|
334
|
+
collector.visit(parsed.tree)
|
|
335
|
+
for node, definition in zip(collector.nodes, collector.definitions, strict=True):
|
|
336
|
+
if node.id in definition_by_id:
|
|
337
|
+
continue
|
|
338
|
+
nodes.append(node)
|
|
339
|
+
definitions.append(definition)
|
|
340
|
+
definition_by_id[node.id] = definition
|
|
341
|
+
|
|
342
|
+
node_by_id = {node.id: node for node in nodes}
|
|
343
|
+
entrypoint_ranks = _entrypoint_ranks(parsed_files, node_by_id)
|
|
344
|
+
nodes = [
|
|
345
|
+
replace(node, entrypoint_rank=entrypoint_ranks.get(node.id)) for node in nodes
|
|
346
|
+
]
|
|
347
|
+
node_by_id = {node.id: node for node in nodes}
|
|
348
|
+
|
|
349
|
+
import_edges: set[Edge] = set()
|
|
350
|
+
module_bindings: dict[str, list[_ImportBinding]] = defaultdict(list)
|
|
351
|
+
scope_bindings: dict[str, list[_ImportBinding]] = defaultdict(list)
|
|
352
|
+
|
|
353
|
+
for parsed in parsed_files:
|
|
354
|
+
if parsed.tree is None:
|
|
355
|
+
continue
|
|
356
|
+
for syntax in ast.walk(parsed.tree):
|
|
357
|
+
if isinstance(syntax, (ast.Import, ast.ImportFrom)):
|
|
358
|
+
edges, _ = _resolve_import(parsed, syntax, modules, node_by_id)
|
|
359
|
+
import_edges.update(edges)
|
|
360
|
+
|
|
361
|
+
module_facts = _ScopeFacts().collect(parsed.tree.body)
|
|
362
|
+
for syntax in module_facts.imports:
|
|
363
|
+
_, bindings = _resolve_import(parsed, syntax, modules, node_by_id)
|
|
364
|
+
module_bindings[parsed.module].extend(bindings)
|
|
365
|
+
|
|
366
|
+
definition_by_id = {definition.node_id: definition for definition in definitions}
|
|
367
|
+
for definition in definitions:
|
|
368
|
+
facts = _ScopeFacts().collect(definition.syntax.body)
|
|
369
|
+
parsed = parsed_by_module[_module_from_node_id(definition.node_id, modules)]
|
|
370
|
+
for syntax in facts.imports:
|
|
371
|
+
_, bindings = _resolve_import(parsed, syntax, modules, node_by_id)
|
|
372
|
+
scope_bindings[definition.node_id].extend(bindings)
|
|
373
|
+
|
|
374
|
+
call_edges: list[Edge] = []
|
|
375
|
+
children_by_parent: dict[str, list[Node]] = defaultdict(list)
|
|
376
|
+
for definition in definitions:
|
|
377
|
+
children_by_parent[definition.parent_id].append(node_by_id[definition.node_id])
|
|
378
|
+
nodes_by_name: dict[str, list[Node]] = defaultdict(list)
|
|
379
|
+
for node in nodes:
|
|
380
|
+
if node.kind != "module":
|
|
381
|
+
nodes_by_name[node.name].append(node)
|
|
382
|
+
|
|
383
|
+
for definition in definitions:
|
|
384
|
+
module = _module_from_node_id(definition.node_id, modules)
|
|
385
|
+
facts = _ScopeFacts().collect(definition.syntax.body)
|
|
386
|
+
bindings = list(module_bindings[module])
|
|
387
|
+
for ancestor_id in definition.function_ancestors:
|
|
388
|
+
bindings.extend(scope_bindings[ancestor_id])
|
|
389
|
+
bindings.extend(scope_bindings[definition.node_id])
|
|
390
|
+
binding_map = {binding.local_name: binding for binding in bindings}
|
|
391
|
+
for call in facts.calls:
|
|
392
|
+
call_edges.extend(
|
|
393
|
+
_resolve_call(
|
|
394
|
+
definition,
|
|
395
|
+
call,
|
|
396
|
+
module,
|
|
397
|
+
binding_map,
|
|
398
|
+
node_by_id,
|
|
399
|
+
nodes_by_name,
|
|
400
|
+
children_by_parent,
|
|
401
|
+
)
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
all_edges = [*import_edges, *call_edges]
|
|
405
|
+
annotations: list[ConceptAnnotation] = []
|
|
406
|
+
for parsed in parsed_files:
|
|
407
|
+
if parsed.tree is None:
|
|
408
|
+
continue
|
|
409
|
+
annotations.extend(
|
|
410
|
+
_concepts_for_target(node_by_id[parsed.module], parsed.tree, parsed.source)
|
|
411
|
+
)
|
|
412
|
+
for definition in definitions:
|
|
413
|
+
annotations.extend(
|
|
414
|
+
_concepts_for_target(
|
|
415
|
+
node_by_id[definition.node_id],
|
|
416
|
+
definition.syntax,
|
|
417
|
+
parsed_by_module[
|
|
418
|
+
_module_from_node_id(definition.node_id, modules)
|
|
419
|
+
].source,
|
|
420
|
+
)
|
|
421
|
+
)
|
|
422
|
+
draft = Graph(
|
|
423
|
+
nodes=tuple(nodes),
|
|
424
|
+
edges=tuple(all_edges),
|
|
425
|
+
entrypoint_candidates=(),
|
|
426
|
+
project_root=str(project_root),
|
|
427
|
+
file_hashes={parsed.relative_path: parsed.digest for parsed in parsed_files},
|
|
428
|
+
concept_annotations=tuple(annotations),
|
|
429
|
+
partial_files=tuple(
|
|
430
|
+
parsed.relative_path for parsed in parsed_files if parsed.tree is None
|
|
431
|
+
),
|
|
432
|
+
)
|
|
433
|
+
try:
|
|
434
|
+
return finalize_graph(draft, entrypoint=entrypoint)
|
|
435
|
+
except GraphFinalizationError as error:
|
|
436
|
+
raise PythonParseError(str(error)) from error
|
|
437
|
+
|
|
438
|
+
def concepts(self, node: Node, source: str) -> list[ConceptAnnotation]:
|
|
439
|
+
"""Return only AST-proven concepts owned by ``node``."""
|
|
440
|
+
|
|
441
|
+
if node.partial:
|
|
442
|
+
return []
|
|
443
|
+
try:
|
|
444
|
+
tree = ast.parse(source, filename=node.file)
|
|
445
|
+
except (SyntaxError, ValueError):
|
|
446
|
+
return []
|
|
447
|
+
target: ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef | None
|
|
448
|
+
target = tree if node.kind == "module" else None
|
|
449
|
+
if target is None:
|
|
450
|
+
expected_types = {
|
|
451
|
+
"class": (ast.ClassDef,),
|
|
452
|
+
"function": (ast.FunctionDef, ast.AsyncFunctionDef),
|
|
453
|
+
}[node.kind]
|
|
454
|
+
target = next(
|
|
455
|
+
(
|
|
456
|
+
candidate
|
|
457
|
+
for candidate in ast.walk(tree)
|
|
458
|
+
if isinstance(candidate, expected_types)
|
|
459
|
+
and candidate.name == node.name
|
|
460
|
+
and candidate.lineno == node.lineno
|
|
461
|
+
),
|
|
462
|
+
None,
|
|
463
|
+
)
|
|
464
|
+
return _concepts_for_target(node, target, source) if target is not None else []
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _discover_python_files(requested: Path) -> tuple[Path, tuple[Path, ...]]:
|
|
468
|
+
normalized = requested.expanduser().resolve()
|
|
469
|
+
try:
|
|
470
|
+
discovery = discover_source_files(normalized, PythonAstAdapter.file_extensions)
|
|
471
|
+
except SourceDiscoveryError as error:
|
|
472
|
+
raise PythonParseError(str(error)) from error
|
|
473
|
+
if not discovery.files:
|
|
474
|
+
if normalized.is_file():
|
|
475
|
+
raise PythonParseError(f"expected a Python file or directory: {normalized}")
|
|
476
|
+
raise PythonParseError(f"no Python files found under: {normalized}")
|
|
477
|
+
return discovery.root, discovery.files
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _parse_file(path: Path, project_root: Path) -> _ParsedFile:
|
|
481
|
+
raw = path.read_bytes()
|
|
482
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
483
|
+
relative = path.relative_to(project_root)
|
|
484
|
+
try:
|
|
485
|
+
with tokenize.open(path) as source_file:
|
|
486
|
+
source = source_file.read()
|
|
487
|
+
tree: ast.Module | None = ast.parse(source, filename=str(path))
|
|
488
|
+
except (SyntaxError, UnicodeDecodeError):
|
|
489
|
+
source = raw.decode("utf-8", errors="replace")
|
|
490
|
+
tree = None
|
|
491
|
+
return _ParsedFile(
|
|
492
|
+
path=path,
|
|
493
|
+
relative_path=relative.as_posix(),
|
|
494
|
+
module=_module_name(relative, project_root),
|
|
495
|
+
source=source,
|
|
496
|
+
digest=digest,
|
|
497
|
+
tree=tree,
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _module_name(relative: Path, project_root: Path) -> str:
|
|
502
|
+
parts = list(relative.with_suffix("").parts)
|
|
503
|
+
if parts[-1] == "__init__":
|
|
504
|
+
parts.pop()
|
|
505
|
+
if (project_root / "__init__.py").is_file():
|
|
506
|
+
parts.insert(0, project_root.name)
|
|
507
|
+
if not parts:
|
|
508
|
+
return project_root.name
|
|
509
|
+
return ".".join(parts)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _module_node(parsed: _ParsedFile) -> Node:
|
|
513
|
+
line_count = max(1, len(parsed.source.splitlines()))
|
|
514
|
+
return Node(
|
|
515
|
+
id=parsed.module,
|
|
516
|
+
kind="module",
|
|
517
|
+
name=parsed.module.rsplit(".", 1)[-1],
|
|
518
|
+
language="python",
|
|
519
|
+
file=parsed.relative_path,
|
|
520
|
+
lineno=1,
|
|
521
|
+
end_lineno=line_count,
|
|
522
|
+
loc=line_count,
|
|
523
|
+
region=_region_for(parsed.module),
|
|
524
|
+
partial=parsed.tree is None,
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _region_for(module: str) -> str:
|
|
529
|
+
return module
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _entrypoint_ranks(
|
|
533
|
+
parsed_files: tuple[_ParsedFile, ...], node_by_id: dict[str, Node]
|
|
534
|
+
) -> dict[str, int]:
|
|
535
|
+
ranks: dict[str, int] = {}
|
|
536
|
+
for parsed in parsed_files:
|
|
537
|
+
if parsed.tree is None:
|
|
538
|
+
continue
|
|
539
|
+
module_rank: int | None = None
|
|
540
|
+
if any(_is_main_guard(statement) for statement in parsed.tree.body):
|
|
541
|
+
module_rank = 0
|
|
542
|
+
for statement in parsed.tree.body:
|
|
543
|
+
if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
544
|
+
candidate_id = f"{parsed.module}.{statement.name}"
|
|
545
|
+
if statement.name == "main" and candidate_id in node_by_id:
|
|
546
|
+
ranks[candidate_id] = min(ranks.get(candidate_id, 1), 1)
|
|
547
|
+
if _is_app_assignment(statement):
|
|
548
|
+
module_rank = min(module_rank if module_rank is not None else 2, 2)
|
|
549
|
+
if parsed.path.name == "__main__.py":
|
|
550
|
+
module_rank = min(module_rank if module_rank is not None else 3, 3)
|
|
551
|
+
if module_rank is not None:
|
|
552
|
+
ranks[parsed.module] = module_rank
|
|
553
|
+
return ranks
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _is_main_guard(statement: ast.stmt) -> bool:
|
|
557
|
+
if not isinstance(statement, ast.If) or not isinstance(statement.test, ast.Compare):
|
|
558
|
+
return False
|
|
559
|
+
comparison = statement.test
|
|
560
|
+
if len(comparison.ops) != 1 or not isinstance(comparison.ops[0], ast.Eq):
|
|
561
|
+
return False
|
|
562
|
+
if len(comparison.comparators) != 1:
|
|
563
|
+
return False
|
|
564
|
+
left, right = comparison.left, comparison.comparators[0]
|
|
565
|
+
return (
|
|
566
|
+
isinstance(left, ast.Name)
|
|
567
|
+
and left.id == "__name__"
|
|
568
|
+
and isinstance(right, ast.Constant)
|
|
569
|
+
and right.value == "__main__"
|
|
570
|
+
) or (
|
|
571
|
+
isinstance(right, ast.Name)
|
|
572
|
+
and right.id == "__name__"
|
|
573
|
+
and isinstance(left, ast.Constant)
|
|
574
|
+
and left.value == "__main__"
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _is_app_assignment(statement: ast.stmt) -> bool:
|
|
579
|
+
target: ast.expr | None = None
|
|
580
|
+
value: ast.expr | None = None
|
|
581
|
+
if isinstance(statement, ast.Assign) and len(statement.targets) == 1:
|
|
582
|
+
target, value = statement.targets[0], statement.value
|
|
583
|
+
elif isinstance(statement, ast.AnnAssign):
|
|
584
|
+
target, value = statement.target, statement.value
|
|
585
|
+
if not isinstance(target, ast.Name) or target.id != "app" or not isinstance(value, ast.Call):
|
|
586
|
+
return False
|
|
587
|
+
factory = _dotted_name(value.func)
|
|
588
|
+
return bool(factory and factory.rsplit(".", 1)[-1] in _APP_FACTORIES)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _resolve_import(
|
|
592
|
+
parsed: _ParsedFile,
|
|
593
|
+
syntax: ast.Import | ast.ImportFrom,
|
|
594
|
+
modules: set[str],
|
|
595
|
+
node_by_id: dict[str, Node],
|
|
596
|
+
) -> tuple[list[Edge], list[_ImportBinding]]:
|
|
597
|
+
edges: list[Edge] = []
|
|
598
|
+
bindings: list[_ImportBinding] = []
|
|
599
|
+
if isinstance(syntax, ast.Import):
|
|
600
|
+
for alias in syntax.names:
|
|
601
|
+
target = alias.name
|
|
602
|
+
project_module = target if target in modules else None
|
|
603
|
+
external = project_module is None
|
|
604
|
+
edge_target = project_module or f"external:{target}"
|
|
605
|
+
edges.append(
|
|
606
|
+
Edge(parsed.module, edge_target, "import", certain=True, lineno=syntax.lineno, external=external)
|
|
607
|
+
)
|
|
608
|
+
local_name = alias.asname or target.split(".", 1)[0]
|
|
609
|
+
binding_target = target if alias.asname else target.split(".", 1)[0]
|
|
610
|
+
binding_external = not any(
|
|
611
|
+
module == binding_target or module.startswith(f"{binding_target}.")
|
|
612
|
+
for module in modules
|
|
613
|
+
)
|
|
614
|
+
bindings.append(_ImportBinding(local_name, binding_target, binding_external))
|
|
615
|
+
return edges, bindings
|
|
616
|
+
|
|
617
|
+
base = _absolute_import_base(parsed, syntax)
|
|
618
|
+
for alias in syntax.names:
|
|
619
|
+
candidate = f"{base}.{alias.name}" if base else alias.name
|
|
620
|
+
edge_module = candidate if candidate in modules else base if base in modules else None
|
|
621
|
+
external = edge_module is None
|
|
622
|
+
edge_target = edge_module or f"external:{base or candidate}"
|
|
623
|
+
edges.append(
|
|
624
|
+
Edge(parsed.module, edge_target, "import", certain=True, lineno=syntax.lineno, external=external)
|
|
625
|
+
)
|
|
626
|
+
target = candidate
|
|
627
|
+
target_external = target not in node_by_id and base not in modules
|
|
628
|
+
bindings.append(_ImportBinding(alias.asname or alias.name, target, target_external))
|
|
629
|
+
return edges, bindings
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def _absolute_import_base(parsed: _ParsedFile, syntax: ast.ImportFrom) -> str:
|
|
633
|
+
if syntax.level == 0:
|
|
634
|
+
return syntax.module or ""
|
|
635
|
+
module_parts = parsed.module.split(".")
|
|
636
|
+
package_parts = module_parts if parsed.path.name == "__init__.py" else module_parts[:-1]
|
|
637
|
+
ascend = syntax.level - 1
|
|
638
|
+
if ascend > len(package_parts):
|
|
639
|
+
return syntax.module or ""
|
|
640
|
+
prefix = package_parts[: len(package_parts) - ascend]
|
|
641
|
+
if syntax.module:
|
|
642
|
+
prefix.extend(syntax.module.split("."))
|
|
643
|
+
return ".".join(prefix)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def _resolve_call(
|
|
647
|
+
definition: _Definition,
|
|
648
|
+
syntax: ast.Call,
|
|
649
|
+
module: str,
|
|
650
|
+
bindings: dict[str, _ImportBinding],
|
|
651
|
+
node_by_id: dict[str, Node],
|
|
652
|
+
nodes_by_name: dict[str, list[Node]],
|
|
653
|
+
children_by_parent: dict[str, list[Node]],
|
|
654
|
+
) -> list[Edge]:
|
|
655
|
+
dotted = _dotted_name(syntax.func)
|
|
656
|
+
name = _call_leaf_name(syntax.func)
|
|
657
|
+
if name is None:
|
|
658
|
+
return [
|
|
659
|
+
Edge(
|
|
660
|
+
definition.node_id,
|
|
661
|
+
f"external:dynamic-call@{syntax.lineno}",
|
|
662
|
+
"call",
|
|
663
|
+
certain=False,
|
|
664
|
+
lineno=syntax.lineno,
|
|
665
|
+
external=True,
|
|
666
|
+
)
|
|
667
|
+
]
|
|
668
|
+
|
|
669
|
+
root = dotted.split(".", 1)[0] if dotted else name
|
|
670
|
+
binding = bindings.get(root)
|
|
671
|
+
if binding is not None:
|
|
672
|
+
suffix = dotted.split(".", 1)[1] if dotted and "." in dotted else ""
|
|
673
|
+
target = f"{binding.target}.{suffix}" if suffix else binding.target
|
|
674
|
+
if target in node_by_id and node_by_id[target].kind != "module":
|
|
675
|
+
return [_call_edge(definition.node_id, target, syntax.lineno, True, False)]
|
|
676
|
+
if binding.external:
|
|
677
|
+
return [
|
|
678
|
+
_call_edge(definition.node_id, f"external:{target}", syntax.lineno, False, True)
|
|
679
|
+
]
|
|
680
|
+
matches = nodes_by_name.get(name, [])
|
|
681
|
+
if matches:
|
|
682
|
+
return [
|
|
683
|
+
_call_edge(definition.node_id, match.id, syntax.lineno, False, False)
|
|
684
|
+
for match in sorted(matches, key=lambda node: node.id)
|
|
685
|
+
]
|
|
686
|
+
return [
|
|
687
|
+
_call_edge(
|
|
688
|
+
definition.node_id, f"unresolved:{target}", syntax.lineno, False, False
|
|
689
|
+
)
|
|
690
|
+
]
|
|
691
|
+
|
|
692
|
+
if isinstance(syntax.func, ast.Attribute) and isinstance(syntax.func.value, ast.Name):
|
|
693
|
+
if syntax.func.value.id in {"self", "cls"} and definition.enclosing_class_id:
|
|
694
|
+
class_matches = [
|
|
695
|
+
node
|
|
696
|
+
for node in children_by_parent[definition.enclosing_class_id]
|
|
697
|
+
if node.name == name
|
|
698
|
+
]
|
|
699
|
+
if len(class_matches) == 1:
|
|
700
|
+
return [
|
|
701
|
+
_call_edge(
|
|
702
|
+
definition.node_id, class_matches[0].id, syntax.lineno, True, False
|
|
703
|
+
)
|
|
704
|
+
]
|
|
705
|
+
|
|
706
|
+
if isinstance(syntax.func, ast.Name):
|
|
707
|
+
lexical_parents = (*reversed(definition.function_ancestors), module)
|
|
708
|
+
for parent_id in lexical_parents:
|
|
709
|
+
matches = [node for node in children_by_parent[parent_id] if node.name == name]
|
|
710
|
+
if matches:
|
|
711
|
+
certain = len(matches) == 1
|
|
712
|
+
return [
|
|
713
|
+
_call_edge(definition.node_id, match.id, syntax.lineno, certain, False)
|
|
714
|
+
for match in sorted(matches, key=lambda node: node.id)
|
|
715
|
+
]
|
|
716
|
+
|
|
717
|
+
matches = nodes_by_name.get(name, [])
|
|
718
|
+
if matches:
|
|
719
|
+
return [
|
|
720
|
+
_call_edge(definition.node_id, match.id, syntax.lineno, False, False)
|
|
721
|
+
for match in sorted(matches, key=lambda node: node.id)
|
|
722
|
+
]
|
|
723
|
+
|
|
724
|
+
external_name = dotted or name
|
|
725
|
+
if name in _BUILTIN_NAMES:
|
|
726
|
+
external_name = f"builtins.{name}"
|
|
727
|
+
return [
|
|
728
|
+
_call_edge(
|
|
729
|
+
definition.node_id,
|
|
730
|
+
f"external:{external_name}",
|
|
731
|
+
syntax.lineno,
|
|
732
|
+
False,
|
|
733
|
+
True,
|
|
734
|
+
)
|
|
735
|
+
]
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _call_edge(src: str, dst: str, lineno: int, certain: bool, external: bool) -> Edge:
|
|
739
|
+
return Edge(src, dst, "call", certain=certain, lineno=lineno, external=external)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def _dotted_name(expression: ast.expr) -> str | None:
|
|
743
|
+
if isinstance(expression, ast.Name):
|
|
744
|
+
return expression.id
|
|
745
|
+
if isinstance(expression, ast.Attribute):
|
|
746
|
+
parent = _dotted_name(expression.value)
|
|
747
|
+
return f"{parent}.{expression.attr}" if parent else expression.attr
|
|
748
|
+
return None
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def _call_leaf_name(expression: ast.expr) -> str | None:
|
|
752
|
+
if isinstance(expression, ast.Name):
|
|
753
|
+
return expression.id
|
|
754
|
+
if isinstance(expression, ast.Attribute):
|
|
755
|
+
return expression.attr
|
|
756
|
+
return None
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _module_from_node_id(node_id: str, modules: set[str]) -> str:
|
|
760
|
+
return max(
|
|
761
|
+
(module for module in modules if node_id == module or node_id.startswith(f"{module}.")),
|
|
762
|
+
key=len,
|
|
763
|
+
)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
__all__ = ["PythonAstAdapter", "PythonParseError"]
|