auryn-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/pyframework/__init__.py +11 -0
- auryn/ingestors/pyframework/schema.py +20 -0
- auryn/ingestors/pyframework/synthesize.py +130 -0
- auryn/ingestors/python/GRAPH_MODEL.md +175 -0
- auryn/ingestors/python/LIMITATIONS.md +95 -0
- auryn/ingestors/python/__init__.py +5 -0
- auryn/ingestors/python/file_parser.py +385 -0
- auryn/ingestors/python/git_sync.py +419 -0
- auryn/ingestors/python/ids.py +160 -0
- auryn/ingestors/python/incremental.py +206 -0
- auryn/ingestors/python/jedi_engine.py +203 -0
- auryn/ingestors/python/plugin.py +49 -0
- auryn/ingestors/python/raw_model.py +110 -0
- auryn/ingestors/python/resolution.py +760 -0
- auryn/ingestors/python/schema.py +78 -0
- auryn_python-0.1.0.dist-info/METADATA +10 -0
- auryn_python-0.1.0.dist-info/RECORD +20 -0
- auryn_python-0.1.0.dist-info/WHEEL +4 -0
- auryn_python-0.1.0.dist-info/entry_points.txt +5 -0
- auryn_python-0.1.0.dist-info/licenses/LICENSE +557 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Python (`ast` engine) binding for the generic incremental resync engine
|
|
2
|
+
(core/incremental_scoped.py, ADR 0002).
|
|
3
|
+
|
|
4
|
+
Everything mechanical -- the git changed-set → graph affected-closure → scoped emit/read/diff
|
|
5
|
+
-- lives in the shared engine. This module supplies only the Python-specific pieces via
|
|
6
|
+
`PYTHON_SPEC`: the id↔scope mapping, the edge classification, and the two genuinely
|
|
7
|
+
language-specific functions -- reconstructing the resolution symbol table from the graph
|
|
8
|
+
(`read_graph_index`) and emitting a subset against it (`resolution.resolve_incremental`).
|
|
9
|
+
|
|
10
|
+
Correctness (== a full recompute) is enforced by tests/ingestors/
|
|
11
|
+
test_python_incremental_equivalence.py. Fast (`ast`) engine only; the deep (jedi) engine's
|
|
12
|
+
transitive type ripples are out of scope. A *newly-satisfiable* import IS covered by the
|
|
13
|
+
module-level IMPORTS breadcrumb; the residual gaps (importing a not-yet-existing module,
|
|
14
|
+
cross-module MRO ordering) are documented on resolve_incremental.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
from collections import defaultdict
|
|
21
|
+
|
|
22
|
+
from auryn.core.graph_client import GraphWriter
|
|
23
|
+
from auryn.core.graph_model import GraphNode
|
|
24
|
+
from auryn.core.incremental_scoped import ChangeSet, ScopedSpec, incremental_sync
|
|
25
|
+
from auryn.ingestors.python import ids, resolution
|
|
26
|
+
from auryn.ingestors.python.file_parser import parse_module
|
|
27
|
+
from auryn.ingestors.python.git_sync import GitPythonProjectConfig, SyncStats
|
|
28
|
+
from auryn.ingestors.python.plugin import EXCLUDED_DIR_NAMES as _EXCLUDED_DIR_NAMES
|
|
29
|
+
from auryn.ingestors.python.schema import NODE_SCHEMAS
|
|
30
|
+
|
|
31
|
+
__all__ = ["ChangeSet", "incremental_ast_sync", "PYTHON_SPEC", "read_graph_index"]
|
|
32
|
+
|
|
33
|
+
_CONTAINMENT_RELS = (
|
|
34
|
+
"HAS_PACKAGE", "HAS_SUBPACKAGE", "HAS_MODULE", "HAS_CLASS", "HAS_FUNCTION",
|
|
35
|
+
"HAS_METHOD", "HAS_NESTED_CLASS", "HAS_ATTRIBUTE", "HAS_PARAMETER",
|
|
36
|
+
)
|
|
37
|
+
_SYMBOL_REF_RELS = ("CALLS", "EXTENDS", "IMPLEMENTS", "RETURNS")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _range(prefix: str) -> tuple[str, str]:
|
|
41
|
+
return prefix, prefix + chr(0xFFFF)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _scope_id(proj_id: str, path: str, package_dirs: frozenset[str] | None = None) -> str:
|
|
45
|
+
"""The id prefix owning all of a file's entities: package_id for an __init__.py, else
|
|
46
|
+
module_id. `package_dirs` is the ingest-wide import-root context (see incremental_scoped's
|
|
47
|
+
build_path_context); it must match what `parse`/`scope_key_for_path` use so the scope id lines
|
|
48
|
+
up with the module id resolution emits."""
|
|
49
|
+
dotted = ids.dotted_module_path(path, package_dirs)
|
|
50
|
+
return ids.package_id(proj_id, dotted) if ids.is_package_init(path) else ids.module_id(proj_id, dotted)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _module_dotted_from_id(node_id: str) -> str | None:
|
|
54
|
+
for marker in ("/module/", "/package/"):
|
|
55
|
+
if marker in node_id:
|
|
56
|
+
rest = node_id.split(marker, 1)[1]
|
|
57
|
+
for sep in (".class.", ".func.", ".method.", ".attr.", ".param."):
|
|
58
|
+
if sep in rest:
|
|
59
|
+
return rest.split(sep, 1)[0]
|
|
60
|
+
return rest
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _python_project_node(project_name, repo_uri, proj_id, target_sha, ref) -> GraphNode:
|
|
65
|
+
return GraphNode(
|
|
66
|
+
"Project",
|
|
67
|
+
{"id": proj_id},
|
|
68
|
+
{"name": project_name, "repo_uri": repo_uri, "commit_sha": target_sha, "ref": ref, "depth": "fast"},
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _python_referenced_names(m) -> set[str]:
|
|
76
|
+
"""A generous superset of the simple names a module references cross-file: call
|
|
77
|
+
names/receivers, tokens of every type annotation and base expression, and import names.
|
|
78
|
+
Over-collecting is safe (extra graph lookups); under-collecting would drop an edge, which
|
|
79
|
+
the equivalence oracle catches. See read_graph_index (v2 targeting)."""
|
|
80
|
+
names: set[str] = set()
|
|
81
|
+
|
|
82
|
+
def add_expr(expr: str | None) -> None:
|
|
83
|
+
if expr:
|
|
84
|
+
names.update(_IDENT.findall(expr))
|
|
85
|
+
|
|
86
|
+
def fn(f) -> None:
|
|
87
|
+
for c in f.calls:
|
|
88
|
+
names.add(c.name)
|
|
89
|
+
if c.receiver:
|
|
90
|
+
names.add(c.receiver)
|
|
91
|
+
add_expr(f.returns)
|
|
92
|
+
for p in f.params:
|
|
93
|
+
add_expr(p.annotation)
|
|
94
|
+
|
|
95
|
+
def cls(c) -> None:
|
|
96
|
+
for b in c.bases:
|
|
97
|
+
add_expr(b)
|
|
98
|
+
for a in c.class_attrs:
|
|
99
|
+
add_expr(a.annotation)
|
|
100
|
+
for meth in c.methods:
|
|
101
|
+
fn(meth)
|
|
102
|
+
for a in meth.self_attrs:
|
|
103
|
+
add_expr(a.annotation)
|
|
104
|
+
for nested in c.nested:
|
|
105
|
+
cls(nested)
|
|
106
|
+
|
|
107
|
+
for f in m.functions:
|
|
108
|
+
fn(f)
|
|
109
|
+
for c in m.classes:
|
|
110
|
+
cls(c)
|
|
111
|
+
for call in m.calls: # module-scope (top-level) calls: same cross-file names a function body has
|
|
112
|
+
names.add(call.name)
|
|
113
|
+
if call.receiver:
|
|
114
|
+
names.add(call.receiver)
|
|
115
|
+
for imp in m.imports:
|
|
116
|
+
names.add(imp.local)
|
|
117
|
+
names.update(imp.target.split("."))
|
|
118
|
+
return names
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def read_graph_index(writer: GraphWriter, proj_id: str, exclude_keys: set[str], names: set[str]) -> resolution.GraphIndex:
|
|
122
|
+
"""Reconstruct resolution's cross-file symbol table from the graph for every module EXCEPT
|
|
123
|
+
`exclude_keys` (the emit set). v2: the big class/callable maps are read only for symbols
|
|
124
|
+
whose simple name is in `names` (what the blast radius references) -- an indexed lookup
|
|
125
|
+
instead of the whole project. The small package/module lists and inheritance `parents`
|
|
126
|
+
(needed transitively for MRO) are still read in full."""
|
|
127
|
+
lo, hi = _range(proj_id)
|
|
128
|
+
index = resolution.GraphIndex()
|
|
129
|
+
name_list = list(names)
|
|
130
|
+
|
|
131
|
+
for (dotted,) in writer.ro_query(
|
|
132
|
+
"MATCH (p:Package) WHERE p.id >= $lo AND p.id < $hi RETURN p.dotted_path", {"lo": lo, "hi": hi}
|
|
133
|
+
).result_set:
|
|
134
|
+
if dotted not in exclude_keys:
|
|
135
|
+
index.packages.add(dotted)
|
|
136
|
+
index.module_dotteds.add(dotted)
|
|
137
|
+
for (dotted,) in writer.ro_query(
|
|
138
|
+
"MATCH (m:Module) WHERE m.id >= $lo AND m.id < $hi RETURN m.dotted_path", {"lo": lo, "hi": hi}
|
|
139
|
+
).result_set:
|
|
140
|
+
if dotted not in exclude_keys:
|
|
141
|
+
index.module_dotteds.add(dotted)
|
|
142
|
+
|
|
143
|
+
if name_list:
|
|
144
|
+
for qname, cid in writer.ro_query(
|
|
145
|
+
"MATCH (c:Class) WHERE c.id >= $lo AND c.id < $hi AND c.name IN $names RETURN c.qualified_name, c.id",
|
|
146
|
+
{"lo": lo, "hi": hi, "names": name_list},
|
|
147
|
+
).result_set:
|
|
148
|
+
if _module_dotted_from_id(cid) not in exclude_keys:
|
|
149
|
+
index.class_by_qname[qname] = cid
|
|
150
|
+
for label in ("Function", "Method"):
|
|
151
|
+
for qname, nid in writer.ro_query(
|
|
152
|
+
f"MATCH (f:{label}) WHERE f.id >= $lo AND f.id < $hi AND f.name IN $names "
|
|
153
|
+
"RETURN f.qualified_name, f.id",
|
|
154
|
+
{"lo": lo, "hi": hi, "names": name_list},
|
|
155
|
+
).result_set:
|
|
156
|
+
if _module_dotted_from_id(nid) not in exclude_keys:
|
|
157
|
+
index.callable_by_qname[qname] = (nid, label)
|
|
158
|
+
|
|
159
|
+
parents: dict[str, list[str]] = defaultdict(list)
|
|
160
|
+
for cid, cq, bq in writer.ro_query(
|
|
161
|
+
"MATCH (c:Class)-[:EXTENDS|IMPLEMENTS]->(b:Class) WHERE c.id >= $lo AND c.id < $hi "
|
|
162
|
+
"RETURN c.id, c.qualified_name, b.qualified_name",
|
|
163
|
+
{"lo": lo, "hi": hi},
|
|
164
|
+
).result_set:
|
|
165
|
+
if _module_dotted_from_id(cid) not in exclude_keys:
|
|
166
|
+
parents[cq].append(bq)
|
|
167
|
+
index.parents = dict(parents)
|
|
168
|
+
return index
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
PYTHON_SPEC = ScopedSpec(
|
|
172
|
+
language="python",
|
|
173
|
+
node_schemas=tuple(NODE_SCHEMAS),
|
|
174
|
+
node_labels=tuple(s.label for s in NODE_SCHEMAS),
|
|
175
|
+
file_suffixes=(".py",),
|
|
176
|
+
excluded_dir_names=frozenset(_EXCLUDED_DIR_NAMES),
|
|
177
|
+
symbol_ref_rels=_SYMBOL_REF_RELS,
|
|
178
|
+
lateral_rels=_SYMBOL_REF_RELS + ("IMPORTS",),
|
|
179
|
+
containment_rels=_CONTAINMENT_RELS,
|
|
180
|
+
targetable_labels=("Function", "Method", "Class"),
|
|
181
|
+
import_rel="IMPORTS",
|
|
182
|
+
import_target_labels=("Module", "Package"),
|
|
183
|
+
project_id=ids.project_id,
|
|
184
|
+
scope_id_for_path=_scope_id,
|
|
185
|
+
scope_key_for_path=ids.dotted_module_path,
|
|
186
|
+
parse=parse_module,
|
|
187
|
+
build_path_context=ids.package_dirs_from_paths,
|
|
188
|
+
is_broken=lambda m: not m.parse_ok, # ast raised SyntaxError -> keep last-good
|
|
189
|
+
referenced_names=_python_referenced_names,
|
|
190
|
+
read_index=read_graph_index,
|
|
191
|
+
resolve_incremental=resolution.resolve_incremental,
|
|
192
|
+
project_node=_python_project_node,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def incremental_ast_sync(writer, config: GitPythonProjectConfig, source, target_sha, changes, ref=None) -> SyncStats:
|
|
197
|
+
"""Thin wrapper over the shared engine that returns Python's SyncStats (the oracle and the
|
|
198
|
+
git_sync routing both go through here / route_incremental → the same engine)."""
|
|
199
|
+
s = incremental_sync(writer, PYTHON_SPEC, source, target_sha, changes,
|
|
200
|
+
project_name=config.project_name, repo_uri=config.repo_uri, ref=ref)
|
|
201
|
+
return SyncStats(
|
|
202
|
+
nodes_added=s.nodes_added, nodes_updated=s.nodes_updated, nodes_removed=s.nodes_removed,
|
|
203
|
+
edges_added=s.edges_added, edges_updated=s.edges_updated, edges_removed=s.edges_removed,
|
|
204
|
+
commit_sha=s.commit_sha, elapsed_seconds=s.elapsed_seconds,
|
|
205
|
+
files_parsed=s.files_parsed, files_cached=s.files_cached,
|
|
206
|
+
)
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Jedi-backed semantic oracle -- the default deep engine for Python.
|
|
2
|
+
|
|
3
|
+
Jedi is a pure-Python, pip-installable static-analysis library (the engine behind many
|
|
4
|
+
IDEs). It runs in-process, so unlike an out-of-process type checker it needs no second
|
|
5
|
+
runtime, no build step, and no vendored toolchain -- `uv sync` gets it. This oracle wraps
|
|
6
|
+
a Jedi `Project` + per-file `Script`s and answers the two queries resolution.py needs:
|
|
7
|
+
inferred types at a position, and the resolved dotted target of a call.
|
|
8
|
+
|
|
9
|
+
Jedi reads source from disk, so the caller materializes a working tree first (git_sync's
|
|
10
|
+
_jedi_sync_at) and hands this oracle that directory; module paths are repo-relative under it.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import jedi
|
|
20
|
+
|
|
21
|
+
from auryn.core.parallel import make_process_pool
|
|
22
|
+
from auryn.ingestors.python.raw_model import RawFunction, RawModule
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class JediOracle:
|
|
28
|
+
"""Implements resolution.SemanticOracle over a materialized source directory."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, source_dir: str):
|
|
31
|
+
self._root = Path(source_dir)
|
|
32
|
+
self._project = jedi.Project(path=str(self._root))
|
|
33
|
+
self._scripts: dict[str, jedi.Script] = {}
|
|
34
|
+
|
|
35
|
+
def _script(self, module_path: str) -> jedi.Script:
|
|
36
|
+
script = self._scripts.get(module_path)
|
|
37
|
+
if script is None:
|
|
38
|
+
script = jedi.Script(path=str(self._root / module_path), project=self._project)
|
|
39
|
+
self._scripts[module_path] = script
|
|
40
|
+
return script
|
|
41
|
+
|
|
42
|
+
def infer_type(self, module_path: str, line: int, col: int) -> str | None:
|
|
43
|
+
"""The inferred type name at (line, col), e.g. 'str' / 'list' / 'Widget'. Jedi
|
|
44
|
+
returns nothing for an un-inferrable name (unannotated param with no usable info),
|
|
45
|
+
which becomes an honest `unresolved` in the graph."""
|
|
46
|
+
if not line:
|
|
47
|
+
return None
|
|
48
|
+
try:
|
|
49
|
+
for d in self._script(module_path).infer(line, col):
|
|
50
|
+
if d.name:
|
|
51
|
+
return d.name
|
|
52
|
+
except Exception as e: # jedi can raise on odd positions; degrade to unresolved
|
|
53
|
+
logger.debug("jedi.infer failed at %s:%d:%d (%s)", module_path, line, col, e)
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
def resolve_call(self, module_path: str, line: int, col: int) -> str | None:
|
|
57
|
+
"""The callee's dotted full name at (line, col) via goto, following imports --
|
|
58
|
+
reaches receiver-typed calls (`conn.execute` -> 'app.db.Connection.execute') as
|
|
59
|
+
well as bare names. Returns None when Jedi can't bind it (dynamic dispatch)."""
|
|
60
|
+
if not line:
|
|
61
|
+
return None
|
|
62
|
+
try:
|
|
63
|
+
for d in self._script(module_path).goto(line, col, follow_imports=True):
|
|
64
|
+
if d.full_name:
|
|
65
|
+
return d.full_name
|
|
66
|
+
except Exception as e:
|
|
67
|
+
logger.debug("jedi.goto failed at %s:%d:%d (%s)", module_path, line, col, e)
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# -- parallel precompute -----------------------------------------------------------
|
|
72
|
+
#
|
|
73
|
+
# Jedi inference at every position is the ingestor's dominant cost (~130x the ast engine).
|
|
74
|
+
# It parallelizes cleanly: each file's queries are independent, so we fan the per-file Jedi
|
|
75
|
+
# work out across a process pool, then hand resolution.py a pure dict-lookup oracle. This is
|
|
76
|
+
# the same shape as the C# ingestor's ProcessPoolExecutor per-file parsing.
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class PrecomputedOracle:
|
|
80
|
+
"""resolution.SemanticOracle backed by maps filled up front (in parallel), so
|
|
81
|
+
resolution itself does no Jedi work -- just O(1) lookups keyed by (path, line, col)."""
|
|
82
|
+
|
|
83
|
+
def __init__(self, types: dict[tuple[str, int, int], str], calls: dict[tuple[str, int, int], str]):
|
|
84
|
+
self._types = types
|
|
85
|
+
self._calls = calls
|
|
86
|
+
|
|
87
|
+
def infer_type(self, module_path: str, line: int, col: int) -> str | None:
|
|
88
|
+
return self._types.get((module_path, line, col))
|
|
89
|
+
|
|
90
|
+
def resolve_call(self, module_path: str, line: int, col: int) -> str | None:
|
|
91
|
+
return self._calls.get((module_path, line, col))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _collect_queries(module: RawModule) -> tuple[list[tuple[int, int]], list[tuple[int, int]]]:
|
|
95
|
+
"""The exact positions resolution.py will query the oracle at for this module -- must
|
|
96
|
+
mirror resolution's oracle calls: infer_type at each *unannotated* attribute/param name,
|
|
97
|
+
resolve_call at each call. (Querying a superset of attributes is harmless -- dedup on the
|
|
98
|
+
resolution side just reads back the ones it needs.)"""
|
|
99
|
+
type_positions: list[tuple[int, int]] = []
|
|
100
|
+
call_positions: list[tuple[int, int]] = []
|
|
101
|
+
|
|
102
|
+
def do_func(fn: RawFunction) -> None:
|
|
103
|
+
for p in fn.params:
|
|
104
|
+
if p.annotation is None:
|
|
105
|
+
type_positions.append((p.name_line, p.name_col))
|
|
106
|
+
for c in fn.calls:
|
|
107
|
+
call_positions.append((c.line, c.col))
|
|
108
|
+
|
|
109
|
+
def do_class(cls) -> None:
|
|
110
|
+
for a in cls.class_attrs:
|
|
111
|
+
if a.annotation is None:
|
|
112
|
+
type_positions.append((a.name_line, a.name_col))
|
|
113
|
+
for method in cls.methods:
|
|
114
|
+
for a in method.self_attrs:
|
|
115
|
+
if a.annotation is None:
|
|
116
|
+
type_positions.append((a.name_line, a.name_col))
|
|
117
|
+
do_func(method)
|
|
118
|
+
for nested in cls.nested:
|
|
119
|
+
do_class(nested)
|
|
120
|
+
|
|
121
|
+
for fn in module.functions:
|
|
122
|
+
do_func(fn)
|
|
123
|
+
for cls in module.classes:
|
|
124
|
+
do_class(cls)
|
|
125
|
+
return type_positions, call_positions
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _infer_one(script: "jedi.Script", type_positions, call_positions):
|
|
129
|
+
types: dict[tuple[int, int], str] = {}
|
|
130
|
+
for line, col in type_positions:
|
|
131
|
+
if not line:
|
|
132
|
+
continue
|
|
133
|
+
try:
|
|
134
|
+
for d in script.infer(line, col):
|
|
135
|
+
if d.name:
|
|
136
|
+
types[(line, col)] = d.name
|
|
137
|
+
break
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
calls: dict[tuple[int, int], str] = {}
|
|
141
|
+
for line, col in call_positions:
|
|
142
|
+
if not line:
|
|
143
|
+
continue
|
|
144
|
+
try:
|
|
145
|
+
for d in script.goto(line, col, follow_imports=True):
|
|
146
|
+
if d.full_name:
|
|
147
|
+
calls[(line, col)] = d.full_name
|
|
148
|
+
break
|
|
149
|
+
except Exception:
|
|
150
|
+
pass
|
|
151
|
+
return types, calls
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _infer_worker(payload):
|
|
155
|
+
"""Runs in a pool worker: answer the queries for a *batch* of files against ONE Jedi
|
|
156
|
+
project, so its in-memory cross-module analysis is reused across the batch (files in a
|
|
157
|
+
batch are same-package neighbours). Takes/returns only picklable primitives."""
|
|
158
|
+
source_dir, batch = payload
|
|
159
|
+
project = jedi.Project(path=source_dir)
|
|
160
|
+
out = []
|
|
161
|
+
for module_path, type_positions, call_positions in batch:
|
|
162
|
+
script = jedi.Script(path=str(Path(source_dir) / module_path), project=project)
|
|
163
|
+
types, calls = _infer_one(script, type_positions, call_positions)
|
|
164
|
+
out.append((module_path, types, calls))
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# Small contiguous batches: big enough that a worker's Jedi project caches shared imports
|
|
169
|
+
# across a handful of same-package files, small enough that there are many more tasks than
|
|
170
|
+
# workers so the pool load-balances dynamically (a few files carry very heavy inference, and
|
|
171
|
+
# a few huge static chunks would let one straggler gate the whole wall-time).
|
|
172
|
+
_BATCH_SIZE = 8
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _batches(items: list, size: int) -> list[list]:
|
|
176
|
+
return [items[i : i + size] for i in range(0, len(items), size)]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def build_precomputed_oracle(source_dir: str, modules: list[RawModule], workers: int | None = None) -> PrecomputedOracle:
|
|
180
|
+
"""Fan per-file Jedi inference across a process pool, then return a lookup-only oracle.
|
|
181
|
+
`workers=1` (or a single file) runs inline, skipping pool overhead."""
|
|
182
|
+
workers = workers or os.cpu_count() or 1
|
|
183
|
+
per_file = [(m.path, *_collect_queries(m)) for m in modules]
|
|
184
|
+
|
|
185
|
+
results: list = []
|
|
186
|
+
if workers <= 1 or len(per_file) <= 1:
|
|
187
|
+
results = _infer_worker((source_dir, per_file))
|
|
188
|
+
else:
|
|
189
|
+
payloads = [(source_dir, batch) for batch in _batches(per_file, _BATCH_SIZE)]
|
|
190
|
+
# Preload this module (it imports jedi) into the forkserver control process so workers
|
|
191
|
+
# inherit the heavy import via COW instead of re-importing it each -- see core.parallel.
|
|
192
|
+
with make_process_pool(workers, preload=[__name__]) as pool:
|
|
193
|
+
for batch_result in pool.map(_infer_worker, payloads):
|
|
194
|
+
results.extend(batch_result)
|
|
195
|
+
|
|
196
|
+
types: dict[tuple[str, int, int], str] = {}
|
|
197
|
+
calls: dict[tuple[str, int, int], str] = {}
|
|
198
|
+
for module_path, file_types, file_calls in results:
|
|
199
|
+
for (line, col), value in file_types.items():
|
|
200
|
+
types[(module_path, line, col)] = value
|
|
201
|
+
for (line, col), value in file_calls.items():
|
|
202
|
+
calls[(module_path, line, col)] = value
|
|
203
|
+
return PrecomputedOracle(types, calls)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Import-light Python language registration -- the `auryn.languages` entry-point target.
|
|
2
|
+
|
|
3
|
+
Importing this module registers the Python GitLanguage but pulls in NO extraction engine
|
|
4
|
+
(neither the stdlib-`ast` pass nor Jedi): the GitLanguage's `sync` is a lazy thunk that
|
|
5
|
+
imports auryn.ingestors.python.git_sync only when a source actually contains `.py`
|
|
6
|
+
files. That is what keeps a repo with no Python from paying Python's (and Jedi's) import
|
|
7
|
+
cost during discovery. Suffix/exclude constants live here (the single source of truth);
|
|
8
|
+
git_sync imports them back.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from auryn.core.registry import GitLanguage, register_git_language
|
|
14
|
+
|
|
15
|
+
LANGUAGE = "python"
|
|
16
|
+
FILE_SUFFIXES = (".py",)
|
|
17
|
+
EXCLUDED_DIR_NAMES = frozenset(
|
|
18
|
+
{
|
|
19
|
+
"__pycache__", ".venv", "venv", ".tox", ".nox", ".mypy_cache", ".pytest_cache",
|
|
20
|
+
".git", "build", "dist", ".eggs", "node_modules", "site-packages",
|
|
21
|
+
}
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _sync(*args, **kwargs):
|
|
26
|
+
"""Fast phase (stdlib `ast`): structural backbone + statically-resolvable
|
|
27
|
+
inheritance/calls, zero extra deps. Imported lazily so discovery stays engine-free."""
|
|
28
|
+
from auryn.ingestors.python.git_sync import ast_language_sync
|
|
29
|
+
|
|
30
|
+
return ast_language_sync(*args, **kwargs)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _deep_sync(*args, **kwargs):
|
|
34
|
+
"""Deep phase (Jedi): layers inferred variable/attribute/param types and resolved
|
|
35
|
+
(incl. receiver-typed) calls over the fast pass, upgrading the same node ids in place."""
|
|
36
|
+
from auryn.ingestors.python.git_sync import jedi_language_sync
|
|
37
|
+
|
|
38
|
+
return jedi_language_sync(*args, **kwargs)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
register_git_language(
|
|
42
|
+
GitLanguage(
|
|
43
|
+
language=LANGUAGE,
|
|
44
|
+
file_suffixes=FILE_SUFFIXES,
|
|
45
|
+
excluded_dir_names=EXCLUDED_DIR_NAMES,
|
|
46
|
+
sync=_sync,
|
|
47
|
+
deep_sync=_deep_sync,
|
|
48
|
+
)
|
|
49
|
+
)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Plain, picklable dataclasses forming the phase-1/phase-2 boundary of the `ast`
|
|
2
|
+
fallback engine (mirrors ingestors/csharp/raw_model.py).
|
|
3
|
+
|
|
4
|
+
file_parser.parse_module() builds these from a stdlib `ast` tree, one per file. No `ast`
|
|
5
|
+
node ever escapes the parser -- only these dataclasses do -- and resolution.py (the
|
|
6
|
+
project-wide second pass) is the only reader.
|
|
7
|
+
|
|
8
|
+
These deliberately carry *syntactic* facts only (plus name positions, so a semantic
|
|
9
|
+
oracle can be queried at them). resolution.py binds names to project-local declarations
|
|
10
|
+
where it statically can; with no oracle (the `ast` engine) everything unannotated is
|
|
11
|
+
`unresolved`, while the `jedi` engine passes an oracle that infers types and resolves
|
|
12
|
+
receiver-typed calls at those positions.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class RawImport:
|
|
22
|
+
"""A binding introduced by an import statement, normalized to (local name -> target).
|
|
23
|
+
|
|
24
|
+
`from a.b import C as D` -> local="D", target="a.b.C", is_module=False
|
|
25
|
+
`import a.b as m` -> local="m", target="a.b", is_module=True
|
|
26
|
+
`import a` -> local="a", target="a", is_module=True
|
|
27
|
+
Relative imports are resolved to absolute `target` at parse time when possible.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
local: str
|
|
31
|
+
target: str
|
|
32
|
+
is_module: bool
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class RawParameter:
|
|
37
|
+
name: str
|
|
38
|
+
annotation: str | None # source text of the annotation, or None if unannotated
|
|
39
|
+
kind: str # positional_only | positional_or_keyword | keyword_only | var_positional | var_keyword
|
|
40
|
+
position: int
|
|
41
|
+
has_default: bool
|
|
42
|
+
name_line: int = 0 # 1-based line / 0-based col of the name, for a semantic oracle to query
|
|
43
|
+
name_col: int = 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class RawAttribute:
|
|
48
|
+
name: str
|
|
49
|
+
annotation: str | None
|
|
50
|
+
defined_in: str # "class_body" | the method name (e.g. "__init__") for a self.x assignment
|
|
51
|
+
is_classvar: bool
|
|
52
|
+
name_line: int = 0
|
|
53
|
+
name_col: int = 0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class RawCall:
|
|
58
|
+
"""A call site, best-effort. The ast engine binds `name` to a project-local function
|
|
59
|
+
only when it can statically (a bare `name()` via imports/same-module, or
|
|
60
|
+
`alias.name()` where alias is an imported project module). The Jedi engine instead
|
|
61
|
+
resolves the callee at (line, col) via inference, which also reaches receiver-typed
|
|
62
|
+
calls (`conn.execute()`)."""
|
|
63
|
+
|
|
64
|
+
name: str # bare Name id, or the attribute name for `receiver.name()`
|
|
65
|
+
receiver: str | None # the receiver's simple name for `receiver.name()`, else None
|
|
66
|
+
is_simple_name: bool # True when the callee is a bare Name (not an attribute access)
|
|
67
|
+
line: int = 0 # position of the callee name, for the Jedi oracle
|
|
68
|
+
col: int = 0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class RawFunction:
|
|
73
|
+
name: str
|
|
74
|
+
params: list[RawParameter] = field(default_factory=list)
|
|
75
|
+
returns: str | None = None # source text of the -> annotation, or None
|
|
76
|
+
decorators: list[str] = field(default_factory=list)
|
|
77
|
+
is_async: bool = False
|
|
78
|
+
is_generator: bool = False
|
|
79
|
+
line_start: int = 0
|
|
80
|
+
line_end: int = 0
|
|
81
|
+
calls: list[RawCall] = field(default_factory=list)
|
|
82
|
+
self_attrs: list[RawAttribute] = field(default_factory=list) # methods only: self.x = ... targets
|
|
83
|
+
source: str = "" # the def's source text (shared `source` property, as C/C#/PL-SQL store it)
|
|
84
|
+
doc: str = "" # shared `doc` property: leading `#` comment block above the def, cleaned
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class RawClass:
|
|
89
|
+
name: str
|
|
90
|
+
bases: list[str] = field(default_factory=list) # source text of each base expression
|
|
91
|
+
decorators: list[str] = field(default_factory=list)
|
|
92
|
+
line_start: int = 0
|
|
93
|
+
line_end: int = 0
|
|
94
|
+
methods: list[RawFunction] = field(default_factory=list)
|
|
95
|
+
source: str = "" # the class's source text (shared `source` property)
|
|
96
|
+
doc: str = "" # shared `doc` property: leading `#` comment block above the class, cleaned
|
|
97
|
+
class_attrs: list[RawAttribute] = field(default_factory=list)
|
|
98
|
+
nested: list["RawClass"] = field(default_factory=list)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass
|
|
102
|
+
class RawModule:
|
|
103
|
+
path: str # repo-relative
|
|
104
|
+
dotted: str
|
|
105
|
+
is_package: bool = False # True for an __init__.py (represents its Package node)
|
|
106
|
+
imports: list[RawImport] = field(default_factory=list)
|
|
107
|
+
classes: list[RawClass] = field(default_factory=list)
|
|
108
|
+
functions: list[RawFunction] = field(default_factory=list)
|
|
109
|
+
calls: list[RawCall] = field(default_factory=list) # module-scope (top-level) call sites
|
|
110
|
+
parse_ok: bool = True # False if the file failed to parse (SyntaxError) -- see file_parser
|