auryn-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,419 @@
1
+ """Git-tracked (and directory) Python ingestion: moves a project's Python subgraph to a
2
+ target commit/tree by fully recomputing the target state and diffing it against the graph
3
+ (graph_sync.compute_delta) -- the same recompute-and-delta model as the C# ingestor, and
4
+ for the same reason (resolution is global, so a source-level file diff can't tell which
5
+ untouched files need re-examining).
6
+
7
+ Two in-process engines, selected like C#'s roslyn/treesitter:
8
+ * "jedi" (default) -- layers Jedi's static inference over the ast structural pass:
9
+ inferred variable/attribute/param types and resolved calls, including receiver-typed
10
+ ones (`conn.execute()`). Jedi is a pure-Python pip dependency; if it's somehow not
11
+ importable we fall back to "ast" automatically.
12
+ * "ast" -- the stdlib-only engine (file_parser + resolution with no oracle): syntax
13
+ backbone plus the inheritance/calls it can statically resolve. Zero extra deps.
14
+
15
+ Registered as a GitLanguage for ".py", so the multi-language endpoint (core/ingest.py)
16
+ owns Python files in both ingest_repository (git) and ingest_directory (local).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import itertools
22
+ import logging
23
+ import os
24
+ import time
25
+ from concurrent.futures import FIRST_COMPLETED, wait
26
+ from dataclasses import dataclass, field
27
+ from pathlib import Path
28
+
29
+ from auryn.core import progress
30
+ from auryn.core.git_source import GitRepoCache, repos_cache_dir
31
+ from auryn.core.parallel import make_process_pool
32
+ from auryn.core.graph_client import GraphWriter
33
+ from auryn.core.graph_model import GraphEdge, GraphNode
34
+ from auryn.core.graph_sync import stream_apply
35
+ from auryn.core.instrument import phase
36
+ from auryn.core.registry import LanguageIngestResult
37
+ from auryn.ingestors.python import resolution
38
+ from auryn.ingestors.python.file_parser import parse_blob, parse_module
39
+ from auryn.ingestors.python.ids import package_dirs_from_paths as ids_pkg_dirs
40
+ from auryn.ingestors.python.ids import project_id
41
+ from auryn.ingestors.python.plugin import EXCLUDED_DIR_NAMES as _EXCLUDED_DIR_NAMES
42
+ from auryn.ingestors.python.schema import NODE_SCHEMAS
43
+
44
+ logger = logging.getLogger(__name__)
45
+
46
+ DEFAULT_ENGINE = "jedi"
47
+ """Extraction engine for git-sync and the ingestion endpoint. "jedi" adds inferred types +
48
+ resolved calls in-process; it falls back to "ast" (zero-dep, stdlib) automatically if Jedi
49
+ isn't importable."""
50
+
51
+ # How deep a graph the two engines produce, for the two-phase short-circuit in sync_at:
52
+ # ast = "fast", jedi = "deep". A graph already at >= the requested depth (same commit) is
53
+ # left untouched. Unknown/missing (a pre-two-phase graph, or None) ranks lowest.
54
+ _DEPTH_RANK = {None: 0, "": 0, "fast": 1, "deep": 2}
55
+
56
+ # _FILE_SUFFIXES / _EXCLUDED_DIR_NAMES are defined once in plugin.py (the import-light
57
+ # registration module) and imported above, so the language's suffixes have a single source
58
+ # of truth shared by the lazy GitLanguage registration and the walkers below.
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class GitPythonProjectConfig:
63
+ repo_uri: str
64
+ project_name: str
65
+ workers: int | None = None
66
+ """Process-pool size for the jedi engine's per-file inference. None = os.cpu_count()."""
67
+
68
+
69
+ @dataclass
70
+ class SyncStats:
71
+ nodes_added: int = 0
72
+ nodes_updated: int = 0
73
+ nodes_removed: int = 0
74
+ edges_added: int = 0
75
+ edges_updated: int = 0
76
+ edges_removed: int = 0
77
+ commit_sha: str = ""
78
+ elapsed_seconds: float = 0.0
79
+ files_parsed: int = 0
80
+ files_cached: int = 0
81
+ broken_files: list[str] = field(default_factory=list)
82
+
83
+
84
+ def sync(
85
+ writer: GraphWriter,
86
+ config: GitPythonProjectConfig,
87
+ ref: str | None,
88
+ cache_root: Path | None = None,
89
+ engine: str = DEFAULT_ENGINE,
90
+ ) -> SyncStats:
91
+ """Ingest (first run) or move (every run after) a git-tracked Python project's graph
92
+ to `ref`. Fetches, resolves `ref` to a commit, then delegates to sync_at()."""
93
+ repo = GitRepoCache(config.repo_uri, repos_cache_dir(cache_root))
94
+ repo.fetch()
95
+ target_sha = repo.resolve_ref(ref)
96
+ return sync_at(writer, config, repo, target_sha, ref, engine)
97
+
98
+
99
+ def sync_at(
100
+ writer: GraphWriter,
101
+ config: GitPythonProjectConfig,
102
+ source,
103
+ target_sha: str,
104
+ ref: str | None,
105
+ engine: str = DEFAULT_ENGINE,
106
+ ) -> SyncStats:
107
+ """The per-commit half of sync(), given an already-prepared `source` (a GitRepoCache
108
+ or a duck-typed LocalDirectorySource) and a resolved `target_sha`."""
109
+ started = time.monotonic()
110
+ proj_id = project_id(config.project_name)
111
+
112
+ # Creating the indexes brings the graph key into existence before the first read
113
+ # (FalkorDB's RO_QUERY rejects an empty key) -- same ordering C# relies on.
114
+ writer.setup_schema(NODE_SCHEMAS)
115
+
116
+ # Depth marker for two-phase ingestion: the fast (ast) pass and the deep (jedi) pass
117
+ # both target the same commit, so a bare commit_sha check would let the fast pass'
118
+ # recorded sha short-circuit the deep upgrade. We record how deep the graph already is
119
+ # ("fast" < "deep") and skip only when the graph is at least as deep as this engine
120
+ # would make it -- so the deep pass still runs right after the fast one, yet a genuine
121
+ # re-ingest of an unchanged, already-deep project is still a no-op.
122
+ current_sha = writer.get_property("Project", {"id": proj_id}, "commit_sha")
123
+ current_depth = writer.get_property("Project", {"id": proj_id}, "depth")
124
+ requested_depth = "deep" if engine == "jedi" else "fast"
125
+ if current_sha == target_sha and _DEPTH_RANK.get(current_depth, 0) >= _DEPTH_RANK[requested_depth]:
126
+ return SyncStats(commit_sha=target_sha, elapsed_seconds=time.monotonic() - started)
127
+
128
+ # Incremental resync (ADR 0002): when a prior ast graph already exists at a *reachable*
129
+ # git commit, move it by processing only the git diff + its graph-derived blast radius,
130
+ # instead of recomputing the whole tree. Guarded tightly -- ast engine, git source, an
131
+ # existing fast-depth graph at a different commit -- and any failure (e.g. the old commit
132
+ # is unreachable in a shallow clone) falls straight back to the full recompute below.
133
+ if engine == "ast" and current_sha and current_sha != target_sha and current_depth == "fast":
134
+ incr = _route(writer, config, source, target_sha, ref, current_sha)
135
+ if incr is not None:
136
+ return _finish(writer, proj_id, source, incr)
137
+
138
+ if engine == "jedi":
139
+ import importlib.util
140
+
141
+ if importlib.util.find_spec("jedi") is None:
142
+ logger.warning("jedi is not installed; falling back to the ast engine")
143
+ else:
144
+ # Is this a genuine same-commit fast->deep UPGRADE (the ast pass just ran at this commit)?
145
+ # Only then does jedi preserve the ast pass's extern:// placeholders (it can't resolve
146
+ # cross-repo and so doesn't re-author them). A standalone jedi resync (no preceding ast pass,
147
+ # e.g. a different commit) is authoritative instead, so it must still diff stale extern edges
148
+ # away -- otherwise a dropped unresolved call leaves a stale placeholder (PR #17 review).
149
+ is_upgrade = current_sha == target_sha and current_depth == "fast"
150
+ result = _jedi_sync_at(writer, config, source, target_sha, ref, proj_id, started,
151
+ diff_extern_edges=not is_upgrade)
152
+ return _finish(writer, proj_id, source, result)
153
+ return _finish(writer, proj_id, source, _ast_sync_at(writer, config, source, target_sha, ref, proj_id, started))
154
+
155
+
156
+ def _route(writer, config, source, target_sha, ref, current_sha) -> SyncStats | None:
157
+ """Attempt an incremental resync via the shared engine's git routing (working-tree or
158
+ bare-clone producer, ADR 0002); return SyncStats or None to fall back to full recompute."""
159
+ from auryn.core.incremental_scoped import route_incremental
160
+ from auryn.ingestors.python.incremental import PYTHON_SPEC
161
+
162
+ s = route_incremental(
163
+ writer, PYTHON_SPEC, source, target_sha, ref, current_sha,
164
+ project_name=config.project_name, repo_uri=config.repo_uri, logger=logger,
165
+ )
166
+ if s is None:
167
+ return None
168
+ return SyncStats(
169
+ nodes_added=s.nodes_added, nodes_updated=s.nodes_updated, nodes_removed=s.nodes_removed,
170
+ edges_added=s.edges_added, edges_updated=s.edges_updated, edges_removed=s.edges_removed,
171
+ commit_sha=s.commit_sha, elapsed_seconds=s.elapsed_seconds,
172
+ files_parsed=s.files_parsed, files_cached=s.files_cached, broken_files=s.broken_files,
173
+ )
174
+
175
+
176
+ def _finish(writer: GraphWriter, proj_id: str, source, result: SyncStats) -> SyncStats:
177
+ """After any non-short-circuit sync, record the working-tree base for the next
178
+ save-triggered incremental resync (a no-op for git/plain-dir sources)."""
179
+ from auryn.core.incremental_scoped import stamp_worktree
180
+ from auryn.core.local_source import WorkingTreeSource
181
+
182
+ if isinstance(source, WorkingTreeSource):
183
+ stamp_worktree(writer, proj_id, source, (".py",))
184
+ return result
185
+
186
+
187
+ def _project_items(
188
+ config: GitPythonProjectConfig, proj_id: str, target_sha: str, ref: str | None, depth: str
189
+ ) -> list[GraphNode | GraphEdge]:
190
+ return [
191
+ GraphNode(
192
+ "Project",
193
+ {"id": proj_id},
194
+ {
195
+ "name": config.project_name,
196
+ "repo_uri": config.repo_uri,
197
+ "commit_sha": target_sha,
198
+ "ref": ref,
199
+ "depth": depth, # "fast" (ast) or "deep" (jedi) -- see the sync_at short-circuit
200
+ },
201
+ )
202
+ ]
203
+
204
+
205
+ # Below this many files, parse+resolve in memory: parsing is CPU-bound and was the ingest's tall
206
+ # pole, so parse once (in parallel) beats the streaming path's re-parse-per-emit (up to 3x/file).
207
+ # Above it, fall back to the memory-bounded serial stream so a very large first ingest keeps peak
208
+ # RSS flat (holding every module's source at once would dominate). Env-tunable for tuning/tests.
209
+ _STREAM_MEMORY_FILES = int(os.environ.get("AURYN_PY_STREAM_FILES", "5000"))
210
+ # Below this, skip the process pool -- worker startup (spawn) costs more than it saves on a handful.
211
+ _PARALLEL_MIN_FILES = 8
212
+ # In-flight parse tasks per worker. Blob bytes are read just-in-time and held only until their task
213
+ # completes, so peak transient blob RAM is bounded to ~(workers * this) files -- not the whole tree
214
+ # at once (what a plain `pool.map`, which submits every task up front, would hold). >1 keeps each
215
+ # worker fed while the previous result is collected.
216
+ _INFLIGHT_PER_WORKER = 3
217
+
218
+
219
+ def _parse_modules(entries, source, pkg_dirs, workers: int) -> list:
220
+ """Parse every (path, blob_sha) entry into a RawModule, preserving input order (so ids stay
221
+ deterministic). Serial for a handful of files or workers==1 (pool startup isn't worth it),
222
+ else parallel across a process pool with a bounded in-flight window.
223
+
224
+ Memory: a blob is read only when its task is submitted and is released once the task finishes,
225
+ so at most ~(workers * _INFLIGHT_PER_WORKER) source blobs are resident at any moment -- the
226
+ parse parallelizes without holding the whole tree's bytes in memory (which `pool.map` would, by
227
+ submitting every task up front). The parsed RawModules do accumulate -- the in-memory resolve
228
+ pass needs them all -- which is the fast path's deliberate floor (see _ast_sync_at)."""
229
+ entries = list(entries)
230
+ if workers <= 1 or len(entries) <= _PARALLEL_MIN_FILES:
231
+ return [parse_module(source.read_blob(sha), path, pkg_dirs) for path, sha in entries]
232
+
233
+ results: list = [None] * len(entries)
234
+ max_inflight = workers * _INFLIGHT_PER_WORKER
235
+ # Preload the parse module (parse_blob's home) into the forkserver control process so workers
236
+ # inherit its import via COW rather than re-importing per worker -- see core.parallel.
237
+ with make_process_pool(workers, preload=["auryn.ingestors.python.file_parser"]) as pool:
238
+ inflight: dict = {} # Future -> result index
239
+ next_idx = 0
240
+
241
+ def submit(i: int) -> None:
242
+ path, sha = entries[i]
243
+ inflight[pool.submit(parse_blob, (path, source.read_blob(sha), pkg_dirs))] = i
244
+
245
+ while next_idx < len(entries) and len(inflight) < max_inflight:
246
+ submit(next_idx)
247
+ next_idx += 1
248
+ while inflight:
249
+ done, _ = wait(inflight, return_when=FIRST_COMPLETED)
250
+ for fut in done:
251
+ results[inflight.pop(fut)] = fut.result() # re-raises a worker error, as pool.map would
252
+ if next_idx < len(entries):
253
+ submit(next_idx)
254
+ next_idx += 1
255
+ return results
256
+
257
+
258
+ def _ast_sync_at(writer, config, source, target_sha, ref, proj_id, started) -> SyncStats:
259
+ """Parse `.py` blobs and resolve with no oracle.
260
+
261
+ Common path: parse every file **once, in parallel**, then resolve in a single in-memory pass
262
+ (like the jedi engine, minus the oracle). Parsing was the tall pole and the streaming path
263
+ below re-parsed each file up to 3x (once to index, twice to emit for the node+edge passes);
264
+ parsing once across a pool removes both the re-parse and the serialization.
265
+
266
+ Fallback (> _STREAM_MEMORY_FILES): the memory-bounded serial stream -- build_plan scans each
267
+ module once into the lightweight index and emit_streaming re-parses one at a time, so a very
268
+ large first ingest never holds every module's source at once. Re-parsing (vs a blob-sha cache)
269
+ keeps each module's relative-import/dotted path correct for the current layout."""
270
+ entries = source.list_files(target_sha, ".py", _EXCLUDED_DIR_NAMES)
271
+ # Package context for import-root detection (`src/` layouts, wrapper dirs): computed once from
272
+ # the whole file list so every module's dotted path is anchored at its true import root (ADR
273
+ # 0004). The incremental engine derives the same context the same way (incremental_scoped).
274
+ pkg_dirs = ids_pkg_dirs([path for path, _ in entries])
275
+ project_items = _project_items(config, proj_id, target_sha, ref, "fast")
276
+
277
+ if len(entries) > _STREAM_MEMORY_FILES:
278
+ def load_fn(path, blob_sha):
279
+ return parse_module(source.read_blob(blob_sha), path, pkg_dirs)
280
+
281
+ with phase("parse"):
282
+ plan = resolution.build_plan(entries, load_fn, proj_id)
283
+
284
+ def make_target(): # re-iterable: stream_apply runs it once for nodes, once for edges
285
+ return itertools.chain(project_items, resolution.emit_streaming(plan, load_fn, proj_id))
286
+
287
+ return _apply(writer, proj_id, make_target, target_sha, started,
288
+ files_parsed=len(entries), broken=plan.broken)
289
+
290
+ workers = config.workers or os.cpu_count() or 1
291
+ with phase("parse"):
292
+ modules = _parse_modules(entries, source, pkg_dirs, workers)
293
+ broken = sorted(m.path for m in modules if not m.parse_ok)
294
+
295
+ def make_target(): # re-iterable: stream_apply runs it once for nodes, once for edges
296
+ return itertools.chain(project_items, resolution.resolve(modules, proj_id))
297
+
298
+ return _apply(writer, proj_id, make_target, target_sha, started,
299
+ files_parsed=len(modules), broken=broken)
300
+
301
+
302
+ def _jedi_sync_at(writer, config, source, target_sha, ref, proj_id, started, *,
303
+ diff_extern_edges: bool = True) -> SyncStats:
304
+ """Materialize a working tree (Jedi reads source from disk and needs the whole project
305
+ to resolve cross-file), parse each file, and resolve with a JediOracle over that dir.
306
+ `working_tree` yields a temp checkout for a git source and the directory itself for a
307
+ local source, so this path is identical either way. `diff_extern_edges` is False only for a
308
+ same-commit fast->deep upgrade (preserve the ast pass's cross-repo placeholders); True for a
309
+ standalone jedi resync so a dropped unresolved call's stale placeholder is still removed."""
310
+ from auryn.ingestors.python.jedi_engine import build_precomputed_oracle
311
+
312
+ entries = source.list_files(target_sha, ".py", _EXCLUDED_DIR_NAMES)
313
+ relpaths = sorted(path for path, _ in entries)
314
+ with source.working_tree(target_sha) as workdir:
315
+ workdir = Path(workdir)
316
+ for rel, blob_sha in entries:
317
+ path = workdir / rel
318
+ if not path.exists():
319
+ path.parent.mkdir(parents=True, exist_ok=True)
320
+ path.write_bytes(source.read_blob(blob_sha))
321
+ pkg_dirs = ids_pkg_dirs(relpaths) # import-root context; see _ast_sync_at
322
+ with phase("parse"):
323
+ modules = []
324
+ for i, rel in enumerate(relpaths, 1):
325
+ progress.log_file(logger, rel, i, len(relpaths))
326
+ modules.append(parse_module((workdir / rel).read_bytes(), rel, pkg_dirs))
327
+ # Fan the (dominant) Jedi inference cost across a process pool up front, inside the
328
+ # `with` -- workers read files that only exist while the working tree is materialized.
329
+ project_items = _project_items(config, proj_id, target_sha, ref, "deep")
330
+ with phase("resolve"):
331
+ oracle = build_precomputed_oracle(str(workdir), modules, workers=config.workers)
332
+
333
+ def make_target(): # re-iterable; resolve() re-queries the precomputed oracle each pass
334
+ return itertools.chain(project_items, resolution.resolve(modules, proj_id, oracle=oracle))
335
+
336
+ broken = sorted(m.path for m in modules if not m.parse_ok)
337
+ # Stay INSIDE the working-tree context: stream_apply re-runs make_target (hence
338
+ # resolve over the oracle) as it writes, and the oracle is anchored to this checkout.
339
+ return _apply(writer, proj_id, make_target, target_sha, started, files_parsed=len(modules),
340
+ broken=broken, diff_extern_edges=diff_extern_edges)
341
+
342
+
343
+ def _apply(writer: GraphWriter, proj_id: str, make_target, target_sha: str, started: float,
344
+ files_parsed: int, broken: list[str] | None = None, *, diff_extern_edges: bool = True) -> SyncStats:
345
+ # Cross-repo reconciliation (ADR 0004): fold placeholder<->definer pairs among the linkages
346
+ # this ingest touched -- targeted index seeks, no graph-wide scan. A no-op unless this graph
347
+ # holds more than one repo.
348
+ from auryn.core.linkage import reconcile_linkages
349
+
350
+ # stream_apply writes the delta without materialising the target graph, and returns the
351
+ # linkage strings collected inline (see graph_sync.stream_apply / core.linkage). The deep (jedi)
352
+ # pass passes diff_extern_edges=False: it can't resolve cross-repo, so it doesn't re-author the
353
+ # fast pass's extern:// placeholders and must not diff them away before reconcile heals them.
354
+ labels = [s.label for s in NODE_SCHEMAS]
355
+ stats, reals, placeholders = stream_apply(writer, proj_id, labels, make_target,
356
+ diff_extern_edges=diff_extern_edges)
357
+ reconcile_linkages(writer, reals, placeholders)
358
+
359
+ return SyncStats(
360
+ nodes_added=stats.nodes_added,
361
+ nodes_updated=stats.nodes_updated,
362
+ nodes_removed=stats.nodes_removed,
363
+ edges_added=stats.edges_added,
364
+ edges_updated=stats.edges_updated,
365
+ edges_removed=stats.edges_removed,
366
+ commit_sha=target_sha,
367
+ elapsed_seconds=time.monotonic() - started,
368
+ files_parsed=files_parsed,
369
+ files_cached=0,
370
+ broken_files=broken or [],
371
+ )
372
+
373
+
374
+ def _git_language_sync(
375
+ writer: GraphWriter,
376
+ repo,
377
+ target_sha: str,
378
+ *,
379
+ engine: str,
380
+ repo_uri: str,
381
+ project_name: str,
382
+ solution_name: str | None,
383
+ ref: str | None,
384
+ cache_root: Path | None = None,
385
+ ) -> LanguageIngestResult:
386
+ """GitLanguage adapter for the multi-language endpoint (Python has no solution/cache
387
+ knobs -- both are accepted for the shared interface and ignored). `engine` picks the
388
+ phase: "ast" (fast) or "jedi" (deep). Both emit identical node ids, so running jedi
389
+ after ast is a pure in-place upgrade."""
390
+ config = GitPythonProjectConfig(repo_uri=repo_uri, project_name=project_name)
391
+ s = sync_at(writer, config, repo, target_sha, ref, engine=engine)
392
+ return LanguageIngestResult(
393
+ nodes_added=s.nodes_added,
394
+ nodes_updated=s.nodes_updated,
395
+ nodes_removed=s.nodes_removed,
396
+ edges_added=s.edges_added,
397
+ edges_updated=s.edges_updated,
398
+ edges_removed=s.edges_removed,
399
+ files_parsed=s.files_parsed,
400
+ files_cached=s.files_cached,
401
+ commit_sha=s.commit_sha,
402
+ elapsed_seconds=s.elapsed_seconds,
403
+ broken_files=s.broken_files,
404
+ )
405
+
406
+
407
+ def ast_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
408
+ """Fast phase entry point (stdlib `ast`) -- the GitLanguage's `sync`. See plugin.py."""
409
+ return _git_language_sync(writer, repo, target_sha, engine="ast", **kwargs)
410
+
411
+
412
+ def jedi_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
413
+ """Deep phase entry point (Jedi) -- the GitLanguage's `deep_sync`. See plugin.py."""
414
+ return _git_language_sync(writer, repo, target_sha, engine="jedi", **kwargs)
415
+
416
+
417
+ # The GitLanguage itself is registered by the import-light plugin.py (the entry-point
418
+ # target), whose lazy `sync`/`deep_sync` thunks call the two adapters above -- so importing
419
+ # this heavy engine module is deferred until a source actually contains `.py` files.
@@ -0,0 +1,160 @@
1
+ """Canonical id scheme for the Python graph (see GRAPH_MODEL.md).
2
+
3
+ Both in-process engines -- `ast` (resolution.py with no oracle) and `jedi` -- key nodes by
4
+ these functions, so their output is interchangeable in one graph (like the C#
5
+ roslyn/treesitter id parity).
6
+
7
+ Ids embed parameter *names*, never inferred types (the id-stability rule): inferred types
8
+ are volatile, so a type in an id would churn the id on every re-sync.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+
14
+ def project_id(project_name: str) -> str:
15
+ return f"python://{project_name}"
16
+
17
+
18
+ def package_dirs_from_paths(paths) -> frozenset[str]:
19
+ """The set of directories that are regular packages -- those containing an ``__init__.py`` --
20
+ given the full list of repo-relative ``.py`` paths in an ingest. The repo root is spelled `""`.
21
+ This is the package context `dotted_module_path` uses to find a file's true import root; it is
22
+ computed once per ingest from the complete file list and threaded through the full AND
23
+ incremental engines identically (so both map a path to the same dotted name)."""
24
+ dirs: set[str] = set()
25
+ for p in paths:
26
+ p = p.replace("\\", "/")
27
+ if p == "__init__.py" or p.endswith("/__init__.py"):
28
+ dirs.add(p[: -len("__init__.py")].rstrip("/"))
29
+ return frozenset(dirs)
30
+
31
+
32
+ def _import_root_index(dirs: list[str], package_dirs: frozenset[str] | None) -> int:
33
+ """Index into a module's ancestor-directory chain `dirs` where the importable dotted name
34
+ begins -- i.e. the first directory that is part of the import path.
35
+
36
+ With `package_dirs` (the set of `__init__.py`-bearing dirs) AND the innermost dir being a
37
+ regular package, walk UP the contiguous package chain and stop at the first ancestor that is
38
+ NOT a package (or not an identifier): that ancestor is the import root. This drops a `src/`
39
+ layout root and any repo/distribution wrapper dir above the top package -- e.g.
40
+ ``src/pkg/mod.py`` -> start at ``pkg``; ``acme-payments/acme_payments/core.py`` -> start at
41
+ ``acme_payments``.
42
+
43
+ Without package evidence (no `__init__.py` context, or a PEP-420 namespace package whose
44
+ innermost dir isn't a regular package) fall back to the always-safe rule: drop the leading run
45
+ up to and including the last non-identifier dir (a segment with a hyphen/space/etc. can never be
46
+ imported, so it and everything above it is above the root), and keep every identifier segment.
47
+ """
48
+ if package_dirs is not None and dirs and "/".join(dirs) in package_dirs and dirs[-1].isidentifier():
49
+ i = len(dirs) - 1
50
+ while i > 0 and dirs[i - 1].isidentifier() and "/".join(dirs[:i]) in package_dirs:
51
+ i -= 1
52
+ return i
53
+ last_non_ident = -1
54
+ for j, seg in enumerate(dirs):
55
+ if not seg.isidentifier():
56
+ last_non_ident = j
57
+ return last_non_ident + 1
58
+
59
+
60
+ def dotted_module_path(rel_path: str, package_dirs: frozenset[str] | None = None) -> str:
61
+ """Repo-relative file path -> importable dotted module name, anchored at the file's true import
62
+ root so the dotted name equals what an importing module writes (the cross-repo linkage key,
63
+ ADR 0004, depends on this).
64
+
65
+ 'app/widget.py' -> 'app.widget'
66
+ 'app/__init__.py' -> 'app' (the package the __init__ defines)
67
+ 'main.py' -> 'main'
68
+ 'acme-payments/acme_pay/x.py' -> 'acme_pay.x' (hyphenated distribution dir isn't importable)
69
+ 'src/pkg/mod.py' (pkg is a package) -> 'pkg.mod' (only with package_dirs: `src/` is the root)
70
+
71
+ `package_dirs` (from `package_dirs_from_paths`) enables `__init__.py`-aware root detection for
72
+ `src/` layouts and identifier-named wrapper dirs. Without it the derivation is context-free (only
73
+ provably-non-importable leading segments are stripped) -- REQUIRED as the fallback because the
74
+ id, scope-key and scope-id derivations must all map a given path to the SAME dotted name, in
75
+ both the full and incremental engines, or a scoped resync would target the wrong nodes (see
76
+ tests/ingestors/test_python_incremental_equivalence.py). The engines pass the same
77
+ ingest-wide `package_dirs` to all three, so they stay in agreement. For a clean or namespace
78
+ layout the two modes produce identical output.
79
+ """
80
+ p = rel_path.replace("\\", "/")
81
+ if p.endswith(".py"):
82
+ p = p[: -len(".py")]
83
+ is_init = False
84
+ if p.endswith("/__init__"):
85
+ p = p[: -len("/__init__")]
86
+ is_init = True
87
+ elif p == "__init__":
88
+ p, is_init = "", True
89
+ p = p.strip("/")
90
+ if not p:
91
+ return ""
92
+ parts = p.split("/")
93
+ # For an __init__, the module IS the package: all components are directories, none is a leaf.
94
+ dirs, leaf = (parts, None) if is_init else (parts[:-1], parts[-1])
95
+ start = _import_root_index(dirs, package_dirs)
96
+ kept = dirs[start:]
97
+ if leaf is not None:
98
+ kept = kept + [leaf]
99
+ return ".".join(kept)
100
+
101
+
102
+ def is_package_init(rel_path: str) -> bool:
103
+ p = rel_path.replace("\\", "/")
104
+ return p == "__init__.py" or p.endswith("/__init__.py")
105
+
106
+
107
+ def package_id(proj_id: str, dotted: str) -> str:
108
+ return f"{proj_id}/package/{dotted}"
109
+
110
+
111
+ def module_id(proj_id: str, dotted: str) -> str:
112
+ return f"{proj_id}/module/{dotted}"
113
+
114
+
115
+ def class_id(scope_id: str, name: str) -> str:
116
+ """`scope_id` is the containing module_id (top-level class) or the parent class_id
117
+ (nested class)."""
118
+ return f"{scope_id}.class.{name.lower()}"
119
+
120
+
121
+ def _sig(param_names: list[str]) -> str:
122
+ return ",".join(param_names)
123
+
124
+
125
+ def function_id(mod_id: str, name: str, param_names: list[str]) -> str:
126
+ return f"{mod_id}.func.{name.lower()}({_sig(param_names)})"
127
+
128
+
129
+ def method_id(cls_id: str, name: str, param_names: list[str]) -> str:
130
+ return f"{cls_id}.method.{name.lower()}({_sig(param_names)})"
131
+
132
+
133
+ def attribute_id(cls_id: str, name: str) -> str:
134
+ return f"{cls_id}.attr.{name.lower()}"
135
+
136
+
137
+ def parameter_id(owner_id: str, position: int, name: str) -> str:
138
+ return f"{owner_id}.param.{position}.{name.lower()}"
139
+
140
+
141
+ # -- cross-repo linkage (ADR 0004) -------------------------------------------------
142
+ # Project-INDEPENDENT identity for a symbol as an external consumer would name it: the
143
+ # module/class path plus the name (never params/types -- a consumer can't know a callee's param
144
+ # names, and the id-stability rule forbids volatile types in ids). It is BOTH the id of the
145
+ # placeholder node that stands in for an unresolved reference AND the `linkage` property stamped
146
+ # on the real, locally-defined node -- so ingesting the defining repo into the same graph lets
147
+ # resolution match placeholder.id == real.linkage and upgrade in place. `::` separates the
148
+ # qualifier from the symbol so it can't collide with a module placeholder's id.
149
+ _EXTERN = "extern://python"
150
+
151
+
152
+ def extern_symbol_key(qualifier: str, name: str) -> str:
153
+ """`qualifier` is the dotted module (top-level function/class) or the class qname (method).
154
+ e.g. ('app.db','get_connection') -> 'extern://python/app.db::get_connection'."""
155
+ return f"{_EXTERN}/{qualifier}::{name}" if qualifier else f"{_EXTERN}/::{name}"
156
+
157
+
158
+ def extern_module_key(dotted: str) -> str:
159
+ """Placeholder id / linkage for an imported module not in this ingest (e.g. 'json')."""
160
+ return f"{_EXTERN}/{dotted}"