auryn-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/pyframework/__init__.py +11 -0
- auryn/ingestors/pyframework/schema.py +20 -0
- auryn/ingestors/pyframework/synthesize.py +130 -0
- auryn/ingestors/python/GRAPH_MODEL.md +175 -0
- auryn/ingestors/python/LIMITATIONS.md +95 -0
- auryn/ingestors/python/__init__.py +5 -0
- auryn/ingestors/python/file_parser.py +385 -0
- auryn/ingestors/python/git_sync.py +419 -0
- auryn/ingestors/python/ids.py +160 -0
- auryn/ingestors/python/incremental.py +206 -0
- auryn/ingestors/python/jedi_engine.py +203 -0
- auryn/ingestors/python/plugin.py +49 -0
- auryn/ingestors/python/raw_model.py +110 -0
- auryn/ingestors/python/resolution.py +760 -0
- auryn/ingestors/python/schema.py +78 -0
- auryn_python-0.1.0.dist-info/METADATA +10 -0
- auryn_python-0.1.0.dist-info/RECORD +20 -0
- auryn_python-0.1.0.dist-info/WHEEL +4 -0
- auryn_python-0.1.0.dist-info/entry_points.txt +5 -0
- auryn_python-0.1.0.dist-info/licenses/LICENSE +557 -0
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"""Git-tracked (and directory) Python ingestion: moves a project's Python subgraph to a
|
|
2
|
+
target commit/tree by fully recomputing the target state and diffing it against the graph
|
|
3
|
+
(graph_sync.compute_delta) -- the same recompute-and-delta model as the C# ingestor, and
|
|
4
|
+
for the same reason (resolution is global, so a source-level file diff can't tell which
|
|
5
|
+
untouched files need re-examining).
|
|
6
|
+
|
|
7
|
+
Two in-process engines, selected like C#'s roslyn/treesitter:
|
|
8
|
+
* "jedi" (default) -- layers Jedi's static inference over the ast structural pass:
|
|
9
|
+
inferred variable/attribute/param types and resolved calls, including receiver-typed
|
|
10
|
+
ones (`conn.execute()`). Jedi is a pure-Python pip dependency; if it's somehow not
|
|
11
|
+
importable we fall back to "ast" automatically.
|
|
12
|
+
* "ast" -- the stdlib-only engine (file_parser + resolution with no oracle): syntax
|
|
13
|
+
backbone plus the inheritance/calls it can statically resolve. Zero extra deps.
|
|
14
|
+
|
|
15
|
+
Registered as a GitLanguage for ".py", so the multi-language endpoint (core/ingest.py)
|
|
16
|
+
owns Python files in both ingest_repository (git) and ingest_directory (local).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import itertools
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
import time
|
|
25
|
+
from concurrent.futures import FIRST_COMPLETED, wait
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from auryn.core import progress
|
|
30
|
+
from auryn.core.git_source import GitRepoCache, repos_cache_dir
|
|
31
|
+
from auryn.core.parallel import make_process_pool
|
|
32
|
+
from auryn.core.graph_client import GraphWriter
|
|
33
|
+
from auryn.core.graph_model import GraphEdge, GraphNode
|
|
34
|
+
from auryn.core.graph_sync import stream_apply
|
|
35
|
+
from auryn.core.instrument import phase
|
|
36
|
+
from auryn.core.registry import LanguageIngestResult
|
|
37
|
+
from auryn.ingestors.python import resolution
|
|
38
|
+
from auryn.ingestors.python.file_parser import parse_blob, parse_module
|
|
39
|
+
from auryn.ingestors.python.ids import package_dirs_from_paths as ids_pkg_dirs
|
|
40
|
+
from auryn.ingestors.python.ids import project_id
|
|
41
|
+
from auryn.ingestors.python.plugin import EXCLUDED_DIR_NAMES as _EXCLUDED_DIR_NAMES
|
|
42
|
+
from auryn.ingestors.python.schema import NODE_SCHEMAS
|
|
43
|
+
|
|
44
|
+
logger = logging.getLogger(__name__)
|
|
45
|
+
|
|
46
|
+
DEFAULT_ENGINE = "jedi"
|
|
47
|
+
"""Extraction engine for git-sync and the ingestion endpoint. "jedi" adds inferred types +
|
|
48
|
+
resolved calls in-process; it falls back to "ast" (zero-dep, stdlib) automatically if Jedi
|
|
49
|
+
isn't importable."""
|
|
50
|
+
|
|
51
|
+
# How deep a graph the two engines produce, for the two-phase short-circuit in sync_at:
|
|
52
|
+
# ast = "fast", jedi = "deep". A graph already at >= the requested depth (same commit) is
|
|
53
|
+
# left untouched. Unknown/missing (a pre-two-phase graph, or None) ranks lowest.
|
|
54
|
+
_DEPTH_RANK = {None: 0, "": 0, "fast": 1, "deep": 2}
|
|
55
|
+
|
|
56
|
+
# _FILE_SUFFIXES / _EXCLUDED_DIR_NAMES are defined once in plugin.py (the import-light
|
|
57
|
+
# registration module) and imported above, so the language's suffixes have a single source
|
|
58
|
+
# of truth shared by the lazy GitLanguage registration and the walkers below.
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class GitPythonProjectConfig:
|
|
63
|
+
repo_uri: str
|
|
64
|
+
project_name: str
|
|
65
|
+
workers: int | None = None
|
|
66
|
+
"""Process-pool size for the jedi engine's per-file inference. None = os.cpu_count()."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class SyncStats:
|
|
71
|
+
nodes_added: int = 0
|
|
72
|
+
nodes_updated: int = 0
|
|
73
|
+
nodes_removed: int = 0
|
|
74
|
+
edges_added: int = 0
|
|
75
|
+
edges_updated: int = 0
|
|
76
|
+
edges_removed: int = 0
|
|
77
|
+
commit_sha: str = ""
|
|
78
|
+
elapsed_seconds: float = 0.0
|
|
79
|
+
files_parsed: int = 0
|
|
80
|
+
files_cached: int = 0
|
|
81
|
+
broken_files: list[str] = field(default_factory=list)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def sync(
|
|
85
|
+
writer: GraphWriter,
|
|
86
|
+
config: GitPythonProjectConfig,
|
|
87
|
+
ref: str | None,
|
|
88
|
+
cache_root: Path | None = None,
|
|
89
|
+
engine: str = DEFAULT_ENGINE,
|
|
90
|
+
) -> SyncStats:
|
|
91
|
+
"""Ingest (first run) or move (every run after) a git-tracked Python project's graph
|
|
92
|
+
to `ref`. Fetches, resolves `ref` to a commit, then delegates to sync_at()."""
|
|
93
|
+
repo = GitRepoCache(config.repo_uri, repos_cache_dir(cache_root))
|
|
94
|
+
repo.fetch()
|
|
95
|
+
target_sha = repo.resolve_ref(ref)
|
|
96
|
+
return sync_at(writer, config, repo, target_sha, ref, engine)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def sync_at(
|
|
100
|
+
writer: GraphWriter,
|
|
101
|
+
config: GitPythonProjectConfig,
|
|
102
|
+
source,
|
|
103
|
+
target_sha: str,
|
|
104
|
+
ref: str | None,
|
|
105
|
+
engine: str = DEFAULT_ENGINE,
|
|
106
|
+
) -> SyncStats:
|
|
107
|
+
"""The per-commit half of sync(), given an already-prepared `source` (a GitRepoCache
|
|
108
|
+
or a duck-typed LocalDirectorySource) and a resolved `target_sha`."""
|
|
109
|
+
started = time.monotonic()
|
|
110
|
+
proj_id = project_id(config.project_name)
|
|
111
|
+
|
|
112
|
+
# Creating the indexes brings the graph key into existence before the first read
|
|
113
|
+
# (FalkorDB's RO_QUERY rejects an empty key) -- same ordering C# relies on.
|
|
114
|
+
writer.setup_schema(NODE_SCHEMAS)
|
|
115
|
+
|
|
116
|
+
# Depth marker for two-phase ingestion: the fast (ast) pass and the deep (jedi) pass
|
|
117
|
+
# both target the same commit, so a bare commit_sha check would let the fast pass'
|
|
118
|
+
# recorded sha short-circuit the deep upgrade. We record how deep the graph already is
|
|
119
|
+
# ("fast" < "deep") and skip only when the graph is at least as deep as this engine
|
|
120
|
+
# would make it -- so the deep pass still runs right after the fast one, yet a genuine
|
|
121
|
+
# re-ingest of an unchanged, already-deep project is still a no-op.
|
|
122
|
+
current_sha = writer.get_property("Project", {"id": proj_id}, "commit_sha")
|
|
123
|
+
current_depth = writer.get_property("Project", {"id": proj_id}, "depth")
|
|
124
|
+
requested_depth = "deep" if engine == "jedi" else "fast"
|
|
125
|
+
if current_sha == target_sha and _DEPTH_RANK.get(current_depth, 0) >= _DEPTH_RANK[requested_depth]:
|
|
126
|
+
return SyncStats(commit_sha=target_sha, elapsed_seconds=time.monotonic() - started)
|
|
127
|
+
|
|
128
|
+
# Incremental resync (ADR 0002): when a prior ast graph already exists at a *reachable*
|
|
129
|
+
# git commit, move it by processing only the git diff + its graph-derived blast radius,
|
|
130
|
+
# instead of recomputing the whole tree. Guarded tightly -- ast engine, git source, an
|
|
131
|
+
# existing fast-depth graph at a different commit -- and any failure (e.g. the old commit
|
|
132
|
+
# is unreachable in a shallow clone) falls straight back to the full recompute below.
|
|
133
|
+
if engine == "ast" and current_sha and current_sha != target_sha and current_depth == "fast":
|
|
134
|
+
incr = _route(writer, config, source, target_sha, ref, current_sha)
|
|
135
|
+
if incr is not None:
|
|
136
|
+
return _finish(writer, proj_id, source, incr)
|
|
137
|
+
|
|
138
|
+
if engine == "jedi":
|
|
139
|
+
import importlib.util
|
|
140
|
+
|
|
141
|
+
if importlib.util.find_spec("jedi") is None:
|
|
142
|
+
logger.warning("jedi is not installed; falling back to the ast engine")
|
|
143
|
+
else:
|
|
144
|
+
# Is this a genuine same-commit fast->deep UPGRADE (the ast pass just ran at this commit)?
|
|
145
|
+
# Only then does jedi preserve the ast pass's extern:// placeholders (it can't resolve
|
|
146
|
+
# cross-repo and so doesn't re-author them). A standalone jedi resync (no preceding ast pass,
|
|
147
|
+
# e.g. a different commit) is authoritative instead, so it must still diff stale extern edges
|
|
148
|
+
# away -- otherwise a dropped unresolved call leaves a stale placeholder (PR #17 review).
|
|
149
|
+
is_upgrade = current_sha == target_sha and current_depth == "fast"
|
|
150
|
+
result = _jedi_sync_at(writer, config, source, target_sha, ref, proj_id, started,
|
|
151
|
+
diff_extern_edges=not is_upgrade)
|
|
152
|
+
return _finish(writer, proj_id, source, result)
|
|
153
|
+
return _finish(writer, proj_id, source, _ast_sync_at(writer, config, source, target_sha, ref, proj_id, started))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _route(writer, config, source, target_sha, ref, current_sha) -> SyncStats | None:
|
|
157
|
+
"""Attempt an incremental resync via the shared engine's git routing (working-tree or
|
|
158
|
+
bare-clone producer, ADR 0002); return SyncStats or None to fall back to full recompute."""
|
|
159
|
+
from auryn.core.incremental_scoped import route_incremental
|
|
160
|
+
from auryn.ingestors.python.incremental import PYTHON_SPEC
|
|
161
|
+
|
|
162
|
+
s = route_incremental(
|
|
163
|
+
writer, PYTHON_SPEC, source, target_sha, ref, current_sha,
|
|
164
|
+
project_name=config.project_name, repo_uri=config.repo_uri, logger=logger,
|
|
165
|
+
)
|
|
166
|
+
if s is None:
|
|
167
|
+
return None
|
|
168
|
+
return SyncStats(
|
|
169
|
+
nodes_added=s.nodes_added, nodes_updated=s.nodes_updated, nodes_removed=s.nodes_removed,
|
|
170
|
+
edges_added=s.edges_added, edges_updated=s.edges_updated, edges_removed=s.edges_removed,
|
|
171
|
+
commit_sha=s.commit_sha, elapsed_seconds=s.elapsed_seconds,
|
|
172
|
+
files_parsed=s.files_parsed, files_cached=s.files_cached, broken_files=s.broken_files,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _finish(writer: GraphWriter, proj_id: str, source, result: SyncStats) -> SyncStats:
|
|
177
|
+
"""After any non-short-circuit sync, record the working-tree base for the next
|
|
178
|
+
save-triggered incremental resync (a no-op for git/plain-dir sources)."""
|
|
179
|
+
from auryn.core.incremental_scoped import stamp_worktree
|
|
180
|
+
from auryn.core.local_source import WorkingTreeSource
|
|
181
|
+
|
|
182
|
+
if isinstance(source, WorkingTreeSource):
|
|
183
|
+
stamp_worktree(writer, proj_id, source, (".py",))
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _project_items(
|
|
188
|
+
config: GitPythonProjectConfig, proj_id: str, target_sha: str, ref: str | None, depth: str
|
|
189
|
+
) -> list[GraphNode | GraphEdge]:
|
|
190
|
+
return [
|
|
191
|
+
GraphNode(
|
|
192
|
+
"Project",
|
|
193
|
+
{"id": proj_id},
|
|
194
|
+
{
|
|
195
|
+
"name": config.project_name,
|
|
196
|
+
"repo_uri": config.repo_uri,
|
|
197
|
+
"commit_sha": target_sha,
|
|
198
|
+
"ref": ref,
|
|
199
|
+
"depth": depth, # "fast" (ast) or "deep" (jedi) -- see the sync_at short-circuit
|
|
200
|
+
},
|
|
201
|
+
)
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# Below this many files, parse+resolve in memory: parsing is CPU-bound and was the ingest's tall
|
|
206
|
+
# pole, so parse once (in parallel) beats the streaming path's re-parse-per-emit (up to 3x/file).
|
|
207
|
+
# Above it, fall back to the memory-bounded serial stream so a very large first ingest keeps peak
|
|
208
|
+
# RSS flat (holding every module's source at once would dominate). Env-tunable for tuning/tests.
|
|
209
|
+
_STREAM_MEMORY_FILES = int(os.environ.get("AURYN_PY_STREAM_FILES", "5000"))
|
|
210
|
+
# Below this, skip the process pool -- worker startup (spawn) costs more than it saves on a handful.
|
|
211
|
+
_PARALLEL_MIN_FILES = 8
|
|
212
|
+
# In-flight parse tasks per worker. Blob bytes are read just-in-time and held only until their task
|
|
213
|
+
# completes, so peak transient blob RAM is bounded to ~(workers * this) files -- not the whole tree
|
|
214
|
+
# at once (what a plain `pool.map`, which submits every task up front, would hold). >1 keeps each
|
|
215
|
+
# worker fed while the previous result is collected.
|
|
216
|
+
_INFLIGHT_PER_WORKER = 3
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _parse_modules(entries, source, pkg_dirs, workers: int) -> list:
|
|
220
|
+
"""Parse every (path, blob_sha) entry into a RawModule, preserving input order (so ids stay
|
|
221
|
+
deterministic). Serial for a handful of files or workers==1 (pool startup isn't worth it),
|
|
222
|
+
else parallel across a process pool with a bounded in-flight window.
|
|
223
|
+
|
|
224
|
+
Memory: a blob is read only when its task is submitted and is released once the task finishes,
|
|
225
|
+
so at most ~(workers * _INFLIGHT_PER_WORKER) source blobs are resident at any moment -- the
|
|
226
|
+
parse parallelizes without holding the whole tree's bytes in memory (which `pool.map` would, by
|
|
227
|
+
submitting every task up front). The parsed RawModules do accumulate -- the in-memory resolve
|
|
228
|
+
pass needs them all -- which is the fast path's deliberate floor (see _ast_sync_at)."""
|
|
229
|
+
entries = list(entries)
|
|
230
|
+
if workers <= 1 or len(entries) <= _PARALLEL_MIN_FILES:
|
|
231
|
+
return [parse_module(source.read_blob(sha), path, pkg_dirs) for path, sha in entries]
|
|
232
|
+
|
|
233
|
+
results: list = [None] * len(entries)
|
|
234
|
+
max_inflight = workers * _INFLIGHT_PER_WORKER
|
|
235
|
+
# Preload the parse module (parse_blob's home) into the forkserver control process so workers
|
|
236
|
+
# inherit its import via COW rather than re-importing per worker -- see core.parallel.
|
|
237
|
+
with make_process_pool(workers, preload=["auryn.ingestors.python.file_parser"]) as pool:
|
|
238
|
+
inflight: dict = {} # Future -> result index
|
|
239
|
+
next_idx = 0
|
|
240
|
+
|
|
241
|
+
def submit(i: int) -> None:
|
|
242
|
+
path, sha = entries[i]
|
|
243
|
+
inflight[pool.submit(parse_blob, (path, source.read_blob(sha), pkg_dirs))] = i
|
|
244
|
+
|
|
245
|
+
while next_idx < len(entries) and len(inflight) < max_inflight:
|
|
246
|
+
submit(next_idx)
|
|
247
|
+
next_idx += 1
|
|
248
|
+
while inflight:
|
|
249
|
+
done, _ = wait(inflight, return_when=FIRST_COMPLETED)
|
|
250
|
+
for fut in done:
|
|
251
|
+
results[inflight.pop(fut)] = fut.result() # re-raises a worker error, as pool.map would
|
|
252
|
+
if next_idx < len(entries):
|
|
253
|
+
submit(next_idx)
|
|
254
|
+
next_idx += 1
|
|
255
|
+
return results
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _ast_sync_at(writer, config, source, target_sha, ref, proj_id, started) -> SyncStats:
|
|
259
|
+
"""Parse `.py` blobs and resolve with no oracle.
|
|
260
|
+
|
|
261
|
+
Common path: parse every file **once, in parallel**, then resolve in a single in-memory pass
|
|
262
|
+
(like the jedi engine, minus the oracle). Parsing was the tall pole and the streaming path
|
|
263
|
+
below re-parsed each file up to 3x (once to index, twice to emit for the node+edge passes);
|
|
264
|
+
parsing once across a pool removes both the re-parse and the serialization.
|
|
265
|
+
|
|
266
|
+
Fallback (> _STREAM_MEMORY_FILES): the memory-bounded serial stream -- build_plan scans each
|
|
267
|
+
module once into the lightweight index and emit_streaming re-parses one at a time, so a very
|
|
268
|
+
large first ingest never holds every module's source at once. Re-parsing (vs a blob-sha cache)
|
|
269
|
+
keeps each module's relative-import/dotted path correct for the current layout."""
|
|
270
|
+
entries = source.list_files(target_sha, ".py", _EXCLUDED_DIR_NAMES)
|
|
271
|
+
# Package context for import-root detection (`src/` layouts, wrapper dirs): computed once from
|
|
272
|
+
# the whole file list so every module's dotted path is anchored at its true import root (ADR
|
|
273
|
+
# 0004). The incremental engine derives the same context the same way (incremental_scoped).
|
|
274
|
+
pkg_dirs = ids_pkg_dirs([path for path, _ in entries])
|
|
275
|
+
project_items = _project_items(config, proj_id, target_sha, ref, "fast")
|
|
276
|
+
|
|
277
|
+
if len(entries) > _STREAM_MEMORY_FILES:
|
|
278
|
+
def load_fn(path, blob_sha):
|
|
279
|
+
return parse_module(source.read_blob(blob_sha), path, pkg_dirs)
|
|
280
|
+
|
|
281
|
+
with phase("parse"):
|
|
282
|
+
plan = resolution.build_plan(entries, load_fn, proj_id)
|
|
283
|
+
|
|
284
|
+
def make_target(): # re-iterable: stream_apply runs it once for nodes, once for edges
|
|
285
|
+
return itertools.chain(project_items, resolution.emit_streaming(plan, load_fn, proj_id))
|
|
286
|
+
|
|
287
|
+
return _apply(writer, proj_id, make_target, target_sha, started,
|
|
288
|
+
files_parsed=len(entries), broken=plan.broken)
|
|
289
|
+
|
|
290
|
+
workers = config.workers or os.cpu_count() or 1
|
|
291
|
+
with phase("parse"):
|
|
292
|
+
modules = _parse_modules(entries, source, pkg_dirs, workers)
|
|
293
|
+
broken = sorted(m.path for m in modules if not m.parse_ok)
|
|
294
|
+
|
|
295
|
+
def make_target(): # re-iterable: stream_apply runs it once for nodes, once for edges
|
|
296
|
+
return itertools.chain(project_items, resolution.resolve(modules, proj_id))
|
|
297
|
+
|
|
298
|
+
return _apply(writer, proj_id, make_target, target_sha, started,
|
|
299
|
+
files_parsed=len(modules), broken=broken)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _jedi_sync_at(writer, config, source, target_sha, ref, proj_id, started, *,
|
|
303
|
+
diff_extern_edges: bool = True) -> SyncStats:
|
|
304
|
+
"""Materialize a working tree (Jedi reads source from disk and needs the whole project
|
|
305
|
+
to resolve cross-file), parse each file, and resolve with a JediOracle over that dir.
|
|
306
|
+
`working_tree` yields a temp checkout for a git source and the directory itself for a
|
|
307
|
+
local source, so this path is identical either way. `diff_extern_edges` is False only for a
|
|
308
|
+
same-commit fast->deep upgrade (preserve the ast pass's cross-repo placeholders); True for a
|
|
309
|
+
standalone jedi resync so a dropped unresolved call's stale placeholder is still removed."""
|
|
310
|
+
from auryn.ingestors.python.jedi_engine import build_precomputed_oracle
|
|
311
|
+
|
|
312
|
+
entries = source.list_files(target_sha, ".py", _EXCLUDED_DIR_NAMES)
|
|
313
|
+
relpaths = sorted(path for path, _ in entries)
|
|
314
|
+
with source.working_tree(target_sha) as workdir:
|
|
315
|
+
workdir = Path(workdir)
|
|
316
|
+
for rel, blob_sha in entries:
|
|
317
|
+
path = workdir / rel
|
|
318
|
+
if not path.exists():
|
|
319
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
320
|
+
path.write_bytes(source.read_blob(blob_sha))
|
|
321
|
+
pkg_dirs = ids_pkg_dirs(relpaths) # import-root context; see _ast_sync_at
|
|
322
|
+
with phase("parse"):
|
|
323
|
+
modules = []
|
|
324
|
+
for i, rel in enumerate(relpaths, 1):
|
|
325
|
+
progress.log_file(logger, rel, i, len(relpaths))
|
|
326
|
+
modules.append(parse_module((workdir / rel).read_bytes(), rel, pkg_dirs))
|
|
327
|
+
# Fan the (dominant) Jedi inference cost across a process pool up front, inside the
|
|
328
|
+
# `with` -- workers read files that only exist while the working tree is materialized.
|
|
329
|
+
project_items = _project_items(config, proj_id, target_sha, ref, "deep")
|
|
330
|
+
with phase("resolve"):
|
|
331
|
+
oracle = build_precomputed_oracle(str(workdir), modules, workers=config.workers)
|
|
332
|
+
|
|
333
|
+
def make_target(): # re-iterable; resolve() re-queries the precomputed oracle each pass
|
|
334
|
+
return itertools.chain(project_items, resolution.resolve(modules, proj_id, oracle=oracle))
|
|
335
|
+
|
|
336
|
+
broken = sorted(m.path for m in modules if not m.parse_ok)
|
|
337
|
+
# Stay INSIDE the working-tree context: stream_apply re-runs make_target (hence
|
|
338
|
+
# resolve over the oracle) as it writes, and the oracle is anchored to this checkout.
|
|
339
|
+
return _apply(writer, proj_id, make_target, target_sha, started, files_parsed=len(modules),
|
|
340
|
+
broken=broken, diff_extern_edges=diff_extern_edges)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _apply(writer: GraphWriter, proj_id: str, make_target, target_sha: str, started: float,
|
|
344
|
+
files_parsed: int, broken: list[str] | None = None, *, diff_extern_edges: bool = True) -> SyncStats:
|
|
345
|
+
# Cross-repo reconciliation (ADR 0004): fold placeholder<->definer pairs among the linkages
|
|
346
|
+
# this ingest touched -- targeted index seeks, no graph-wide scan. A no-op unless this graph
|
|
347
|
+
# holds more than one repo.
|
|
348
|
+
from auryn.core.linkage import reconcile_linkages
|
|
349
|
+
|
|
350
|
+
# stream_apply writes the delta without materialising the target graph, and returns the
|
|
351
|
+
# linkage strings collected inline (see graph_sync.stream_apply / core.linkage). The deep (jedi)
|
|
352
|
+
# pass passes diff_extern_edges=False: it can't resolve cross-repo, so it doesn't re-author the
|
|
353
|
+
# fast pass's extern:// placeholders and must not diff them away before reconcile heals them.
|
|
354
|
+
labels = [s.label for s in NODE_SCHEMAS]
|
|
355
|
+
stats, reals, placeholders = stream_apply(writer, proj_id, labels, make_target,
|
|
356
|
+
diff_extern_edges=diff_extern_edges)
|
|
357
|
+
reconcile_linkages(writer, reals, placeholders)
|
|
358
|
+
|
|
359
|
+
return SyncStats(
|
|
360
|
+
nodes_added=stats.nodes_added,
|
|
361
|
+
nodes_updated=stats.nodes_updated,
|
|
362
|
+
nodes_removed=stats.nodes_removed,
|
|
363
|
+
edges_added=stats.edges_added,
|
|
364
|
+
edges_updated=stats.edges_updated,
|
|
365
|
+
edges_removed=stats.edges_removed,
|
|
366
|
+
commit_sha=target_sha,
|
|
367
|
+
elapsed_seconds=time.monotonic() - started,
|
|
368
|
+
files_parsed=files_parsed,
|
|
369
|
+
files_cached=0,
|
|
370
|
+
broken_files=broken or [],
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _git_language_sync(
|
|
375
|
+
writer: GraphWriter,
|
|
376
|
+
repo,
|
|
377
|
+
target_sha: str,
|
|
378
|
+
*,
|
|
379
|
+
engine: str,
|
|
380
|
+
repo_uri: str,
|
|
381
|
+
project_name: str,
|
|
382
|
+
solution_name: str | None,
|
|
383
|
+
ref: str | None,
|
|
384
|
+
cache_root: Path | None = None,
|
|
385
|
+
) -> LanguageIngestResult:
|
|
386
|
+
"""GitLanguage adapter for the multi-language endpoint (Python has no solution/cache
|
|
387
|
+
knobs -- both are accepted for the shared interface and ignored). `engine` picks the
|
|
388
|
+
phase: "ast" (fast) or "jedi" (deep). Both emit identical node ids, so running jedi
|
|
389
|
+
after ast is a pure in-place upgrade."""
|
|
390
|
+
config = GitPythonProjectConfig(repo_uri=repo_uri, project_name=project_name)
|
|
391
|
+
s = sync_at(writer, config, repo, target_sha, ref, engine=engine)
|
|
392
|
+
return LanguageIngestResult(
|
|
393
|
+
nodes_added=s.nodes_added,
|
|
394
|
+
nodes_updated=s.nodes_updated,
|
|
395
|
+
nodes_removed=s.nodes_removed,
|
|
396
|
+
edges_added=s.edges_added,
|
|
397
|
+
edges_updated=s.edges_updated,
|
|
398
|
+
edges_removed=s.edges_removed,
|
|
399
|
+
files_parsed=s.files_parsed,
|
|
400
|
+
files_cached=s.files_cached,
|
|
401
|
+
commit_sha=s.commit_sha,
|
|
402
|
+
elapsed_seconds=s.elapsed_seconds,
|
|
403
|
+
broken_files=s.broken_files,
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def ast_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
|
|
408
|
+
"""Fast phase entry point (stdlib `ast`) -- the GitLanguage's `sync`. See plugin.py."""
|
|
409
|
+
return _git_language_sync(writer, repo, target_sha, engine="ast", **kwargs)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def jedi_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
|
|
413
|
+
"""Deep phase entry point (Jedi) -- the GitLanguage's `deep_sync`. See plugin.py."""
|
|
414
|
+
return _git_language_sync(writer, repo, target_sha, engine="jedi", **kwargs)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# The GitLanguage itself is registered by the import-light plugin.py (the entry-point
|
|
418
|
+
# target), whose lazy `sync`/`deep_sync` thunks call the two adapters above -- so importing
|
|
419
|
+
# this heavy engine module is deferred until a source actually contains `.py` files.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Canonical id scheme for the Python graph (see GRAPH_MODEL.md).
|
|
2
|
+
|
|
3
|
+
Both in-process engines -- `ast` (resolution.py with no oracle) and `jedi` -- key nodes by
|
|
4
|
+
these functions, so their output is interchangeable in one graph (like the C#
|
|
5
|
+
roslyn/treesitter id parity).
|
|
6
|
+
|
|
7
|
+
Ids embed parameter *names*, never inferred types (the id-stability rule): inferred types
|
|
8
|
+
are volatile, so a type in an id would churn the id on every re-sync.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def project_id(project_name: str) -> str:
|
|
15
|
+
return f"python://{project_name}"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def package_dirs_from_paths(paths) -> frozenset[str]:
|
|
19
|
+
"""The set of directories that are regular packages -- those containing an ``__init__.py`` --
|
|
20
|
+
given the full list of repo-relative ``.py`` paths in an ingest. The repo root is spelled `""`.
|
|
21
|
+
This is the package context `dotted_module_path` uses to find a file's true import root; it is
|
|
22
|
+
computed once per ingest from the complete file list and threaded through the full AND
|
|
23
|
+
incremental engines identically (so both map a path to the same dotted name)."""
|
|
24
|
+
dirs: set[str] = set()
|
|
25
|
+
for p in paths:
|
|
26
|
+
p = p.replace("\\", "/")
|
|
27
|
+
if p == "__init__.py" or p.endswith("/__init__.py"):
|
|
28
|
+
dirs.add(p[: -len("__init__.py")].rstrip("/"))
|
|
29
|
+
return frozenset(dirs)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _import_root_index(dirs: list[str], package_dirs: frozenset[str] | None) -> int:
|
|
33
|
+
"""Index into a module's ancestor-directory chain `dirs` where the importable dotted name
|
|
34
|
+
begins -- i.e. the first directory that is part of the import path.
|
|
35
|
+
|
|
36
|
+
With `package_dirs` (the set of `__init__.py`-bearing dirs) AND the innermost dir being a
|
|
37
|
+
regular package, walk UP the contiguous package chain and stop at the first ancestor that is
|
|
38
|
+
NOT a package (or not an identifier): that ancestor is the import root. This drops a `src/`
|
|
39
|
+
layout root and any repo/distribution wrapper dir above the top package -- e.g.
|
|
40
|
+
``src/pkg/mod.py`` -> start at ``pkg``; ``acme-payments/acme_payments/core.py`` -> start at
|
|
41
|
+
``acme_payments``.
|
|
42
|
+
|
|
43
|
+
Without package evidence (no `__init__.py` context, or a PEP-420 namespace package whose
|
|
44
|
+
innermost dir isn't a regular package) fall back to the always-safe rule: drop the leading run
|
|
45
|
+
up to and including the last non-identifier dir (a segment with a hyphen/space/etc. can never be
|
|
46
|
+
imported, so it and everything above it is above the root), and keep every identifier segment.
|
|
47
|
+
"""
|
|
48
|
+
if package_dirs is not None and dirs and "/".join(dirs) in package_dirs and dirs[-1].isidentifier():
|
|
49
|
+
i = len(dirs) - 1
|
|
50
|
+
while i > 0 and dirs[i - 1].isidentifier() and "/".join(dirs[:i]) in package_dirs:
|
|
51
|
+
i -= 1
|
|
52
|
+
return i
|
|
53
|
+
last_non_ident = -1
|
|
54
|
+
for j, seg in enumerate(dirs):
|
|
55
|
+
if not seg.isidentifier():
|
|
56
|
+
last_non_ident = j
|
|
57
|
+
return last_non_ident + 1
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def dotted_module_path(rel_path: str, package_dirs: frozenset[str] | None = None) -> str:
|
|
61
|
+
"""Repo-relative file path -> importable dotted module name, anchored at the file's true import
|
|
62
|
+
root so the dotted name equals what an importing module writes (the cross-repo linkage key,
|
|
63
|
+
ADR 0004, depends on this).
|
|
64
|
+
|
|
65
|
+
'app/widget.py' -> 'app.widget'
|
|
66
|
+
'app/__init__.py' -> 'app' (the package the __init__ defines)
|
|
67
|
+
'main.py' -> 'main'
|
|
68
|
+
'acme-payments/acme_pay/x.py' -> 'acme_pay.x' (hyphenated distribution dir isn't importable)
|
|
69
|
+
'src/pkg/mod.py' (pkg is a package) -> 'pkg.mod' (only with package_dirs: `src/` is the root)
|
|
70
|
+
|
|
71
|
+
`package_dirs` (from `package_dirs_from_paths`) enables `__init__.py`-aware root detection for
|
|
72
|
+
`src/` layouts and identifier-named wrapper dirs. Without it the derivation is context-free (only
|
|
73
|
+
provably-non-importable leading segments are stripped) -- REQUIRED as the fallback because the
|
|
74
|
+
id, scope-key and scope-id derivations must all map a given path to the SAME dotted name, in
|
|
75
|
+
both the full and incremental engines, or a scoped resync would target the wrong nodes (see
|
|
76
|
+
tests/ingestors/test_python_incremental_equivalence.py). The engines pass the same
|
|
77
|
+
ingest-wide `package_dirs` to all three, so they stay in agreement. For a clean or namespace
|
|
78
|
+
layout the two modes produce identical output.
|
|
79
|
+
"""
|
|
80
|
+
p = rel_path.replace("\\", "/")
|
|
81
|
+
if p.endswith(".py"):
|
|
82
|
+
p = p[: -len(".py")]
|
|
83
|
+
is_init = False
|
|
84
|
+
if p.endswith("/__init__"):
|
|
85
|
+
p = p[: -len("/__init__")]
|
|
86
|
+
is_init = True
|
|
87
|
+
elif p == "__init__":
|
|
88
|
+
p, is_init = "", True
|
|
89
|
+
p = p.strip("/")
|
|
90
|
+
if not p:
|
|
91
|
+
return ""
|
|
92
|
+
parts = p.split("/")
|
|
93
|
+
# For an __init__, the module IS the package: all components are directories, none is a leaf.
|
|
94
|
+
dirs, leaf = (parts, None) if is_init else (parts[:-1], parts[-1])
|
|
95
|
+
start = _import_root_index(dirs, package_dirs)
|
|
96
|
+
kept = dirs[start:]
|
|
97
|
+
if leaf is not None:
|
|
98
|
+
kept = kept + [leaf]
|
|
99
|
+
return ".".join(kept)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def is_package_init(rel_path: str) -> bool:
|
|
103
|
+
p = rel_path.replace("\\", "/")
|
|
104
|
+
return p == "__init__.py" or p.endswith("/__init__.py")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def package_id(proj_id: str, dotted: str) -> str:
|
|
108
|
+
return f"{proj_id}/package/{dotted}"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def module_id(proj_id: str, dotted: str) -> str:
|
|
112
|
+
return f"{proj_id}/module/{dotted}"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def class_id(scope_id: str, name: str) -> str:
|
|
116
|
+
"""`scope_id` is the containing module_id (top-level class) or the parent class_id
|
|
117
|
+
(nested class)."""
|
|
118
|
+
return f"{scope_id}.class.{name.lower()}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _sig(param_names: list[str]) -> str:
|
|
122
|
+
return ",".join(param_names)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def function_id(mod_id: str, name: str, param_names: list[str]) -> str:
|
|
126
|
+
return f"{mod_id}.func.{name.lower()}({_sig(param_names)})"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def method_id(cls_id: str, name: str, param_names: list[str]) -> str:
|
|
130
|
+
return f"{cls_id}.method.{name.lower()}({_sig(param_names)})"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def attribute_id(cls_id: str, name: str) -> str:
|
|
134
|
+
return f"{cls_id}.attr.{name.lower()}"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def parameter_id(owner_id: str, position: int, name: str) -> str:
|
|
138
|
+
return f"{owner_id}.param.{position}.{name.lower()}"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# -- cross-repo linkage (ADR 0004) -------------------------------------------------
|
|
142
|
+
# Project-INDEPENDENT identity for a symbol as an external consumer would name it: the
|
|
143
|
+
# module/class path plus the name (never params/types -- a consumer can't know a callee's param
|
|
144
|
+
# names, and the id-stability rule forbids volatile types in ids). It is BOTH the id of the
|
|
145
|
+
# placeholder node that stands in for an unresolved reference AND the `linkage` property stamped
|
|
146
|
+
# on the real, locally-defined node -- so ingesting the defining repo into the same graph lets
|
|
147
|
+
# resolution match placeholder.id == real.linkage and upgrade in place. `::` separates the
|
|
148
|
+
# qualifier from the symbol so it can't collide with a module placeholder's id.
|
|
149
|
+
_EXTERN = "extern://python"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def extern_symbol_key(qualifier: str, name: str) -> str:
|
|
153
|
+
"""`qualifier` is the dotted module (top-level function/class) or the class qname (method).
|
|
154
|
+
e.g. ('app.db','get_connection') -> 'extern://python/app.db::get_connection'."""
|
|
155
|
+
return f"{_EXTERN}/{qualifier}::{name}" if qualifier else f"{_EXTERN}/::{name}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def extern_module_key(dotted: str) -> str:
|
|
159
|
+
"""Placeholder id / linkage for an imported module not in this ingest (e.g. 'json')."""
|
|
160
|
+
return f"{_EXTERN}/{dotted}"
|