codegraph-network 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
codegraph/store.py ADDED
@@ -0,0 +1,638 @@
1
+ """SQLite graph store: symbols, resolved edges, file-level imports, FTS search."""
2
+ from __future__ import annotations
3
+
4
+ import posixpath
5
+ import re
6
+ import sqlite3
7
+ from collections import defaultdict
8
+ from pathlib import Path
9
+
10
+ from .parser import Parsed
11
+ from .rank import pagerank
12
+
13
+ SCHEMA_VERSION = 3
14
+
15
+ SCHEMA = """
16
+ CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY, lang TEXT, hash TEXT, loc INT);
17
+ CREATE TABLE IF NOT EXISTS symbols(
18
+ id INTEGER PRIMARY KEY, file TEXT, name TEXT, qualname TEXT, kind TEXT,
19
+ line_start INT, line_end INT, sig TEXT, doc TEXT, parent INT, rank REAL DEFAULT 0,
20
+ community INT DEFAULT -1);
21
+ CREATE TABLE IF NOT EXISTS raw_edges(src INT, name TEXT, kind TEXT);
22
+ -- conf: how the edge was resolved. extracted = unique target named by the source itself (same file,
23
+ -- import, self/this); inferred = name match by proximity; ambiguous = several equally plausible targets
24
+ CREATE TABLE IF NOT EXISTS edges(src INT, dst INT, kind TEXT, conf TEXT DEFAULT 'inferred');
25
+ CREATE TABLE IF NOT EXISTS raw_imports(file TEXT, raw TEXT);
26
+ CREATE TABLE IF NOT EXISTS imports(file TEXT, dst TEXT);
27
+ CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
28
+ CREATE INDEX IF NOT EXISTS ix_sym_name ON symbols(name);
29
+ CREATE INDEX IF NOT EXISTS ix_sym_file ON symbols(file);
30
+ CREATE INDEX IF NOT EXISTS ix_sym_qual ON symbols(qualname);
31
+ CREATE INDEX IF NOT EXISTS ix_e_src ON edges(src);
32
+ CREATE INDEX IF NOT EXISTS ix_e_dst ON edges(dst);
33
+ CREATE INDEX IF NOT EXISTS ix_re_src ON raw_edges(src);
34
+ CREATE INDEX IF NOT EXISTS ix_imp_file ON imports(file);
35
+ CREATE INDEX IF NOT EXISTS ix_imp_dst ON imports(dst);
36
+ CREATE TABLE IF NOT EXISTS raw_keys(src INT, ns TEXT, key TEXT, role TEXT);
37
+ CREATE INDEX IF NOT EXISTS ix_rk_key ON raw_keys(ns, key);
38
+ CREATE INDEX IF NOT EXISTS ix_rk_src ON raw_keys(src);
39
+ -- non-structural symbol links (similar / tests / key). Kept apart from `edges` so callers, callees,
40
+ -- impact and global PageRank stay purely structural.
41
+ CREATE TABLE IF NOT EXISTS links(src INT, dst INT, kind TEXT, weight REAL, note TEXT);
42
+ CREATE INDEX IF NOT EXISTS ix_l_src ON links(src);
43
+ CREATE INDEX IF NOT EXISTS ix_l_dst ON links(dst);
44
+ CREATE TABLE IF NOT EXISTS file_links(a TEXT, b TEXT, kind TEXT, weight REAL);
45
+ CREATE INDEX IF NOT EXISTS ix_fl_a ON file_links(a);
46
+ CREATE INDEX IF NOT EXISTS ix_fl_b ON file_links(b);
47
+ -- embeddings: vectors are cached by content hash, so only changed symbols are ever re-embedded
48
+ CREATE TABLE IF NOT EXISTS sym_vec(sym_id INTEGER PRIMARY KEY, hash TEXT);
49
+ CREATE TABLE IF NOT EXISTS vec_cache(hash TEXT PRIMARY KEY, vec BLOB);
50
+ CREATE VIRTUAL TABLE IF NOT EXISTS sym_fts USING fts5(name, words, qualname, doc);
51
+ """
52
+
53
+
54
+ def _migration_1(db: sqlite3.Connection) -> bool:
55
+ """Add the rank column (absent in databases written before ranking) and speed up name lookups."""
56
+ db.execute("CREATE INDEX IF NOT EXISTS ix_re_name ON raw_edges(name)")
57
+ cols = {r[1] for r in db.execute("PRAGMA table_info(symbols)")}
58
+ if "rank" in cols:
59
+ return False
60
+ db.execute("ALTER TABLE symbols ADD COLUMN rank REAL DEFAULT 0")
61
+ return True
62
+
63
+
64
+ def _migration_2(db: sqlite3.Connection) -> bool:
65
+ """Links, keys and embedding tables: created by SCHEMA, which is idempotent. Nothing to alter."""
66
+ return False
67
+
68
+
69
+ def _migration_3(db: sqlite3.Connection) -> bool:
70
+ """Edge confidence and symbol community. Old databases are re-indexed (parser version bump)."""
71
+ changed = False
72
+ for table, col, ddl in (("edges", "conf", "TEXT DEFAULT 'inferred'"),
73
+ ("symbols", "community", "INT DEFAULT -1")):
74
+ if col not in {r[1] for r in db.execute(f"PRAGMA table_info({table})")}:
75
+ db.execute(f"ALTER TABLE {table} ADD COLUMN {col} {ddl}")
76
+ changed = True
77
+ return changed
78
+
79
+
80
+ MIGRATIONS = {1: _migration_1, 2: _migration_2, 3: _migration_3}
81
+
82
+ TARGET_KINDS_CALL = {"function", "method", "class", "struct", "type", "interface", "trait", "enum",
83
+ "table", "view", "procedure"} # SQL objects, referenced from docs and other SQL
84
+ TARGET_KINDS_INHERIT = {"class", "interface", "trait", "struct", "type", "enum"}
85
+ CLASS_KINDS = {"class", "struct", "trait", "interface", "impl"}
86
+ # kinds that are containers: a `function` inside one is really a method
87
+ CONTAINER_KINDS = {"class", "struct", "trait", "interface", "impl"}
88
+ CONF_RANK = {"ambiguous": 0, "inferred": 1, "extracted": 2}
89
+ MAX_CANDIDATES = 4 # names defined in more places than this are too ambiguous to link
90
+
91
+ # Python builtins called directly (`isinstance(x, T)`) are not calls to anything in the repo.
92
+ # Without this, a repo method that happens to be called `isinstance` becomes a hub with thousands
93
+ # of phantom in-edges and dominates the global PageRank. A same-file definition still wins, so
94
+ # code that deliberately shadows a builtin keeps working.
95
+ PY_BUILTINS = frozenset(
96
+ "abs all any ascii bin bool breakpoint bytearray bytes callable chr classmethod compile complex "
97
+ "dict dir divmod enumerate eval exec filter float format frozenset getattr globals hasattr hash "
98
+ "help hex id input int isinstance issubclass iter len list locals map max memoryview min next "
99
+ "object oct open ord pow print property range repr reversed round set setattr slice sorted "
100
+ "staticmethod str sum super tuple type vars zip".split()
101
+ )
102
+
103
+
104
+ def split_words(name: str) -> str:
105
+ return re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", name).replace("_", " ")
106
+
107
+
108
+ def _ph(values) -> str:
109
+ """Placeholder list for an IN clause. Safe for empty input."""
110
+ vals = list(values)
111
+ return ",".join("?" * len(vals)) if vals else "NULL"
112
+
113
+
114
+ def _chunks(items, n: int = 500):
115
+ for i in range(0, len(items), n):
116
+ yield items[i:i + n]
117
+
118
+
119
+ def _suffix_paths(path: str) -> list[str]:
120
+ """Every trailing sub-path of `path`, each exactly once: 'a/b/c' -> ['a/b/c', 'b/c', 'c']."""
121
+ parts = [p for p in path.split("/") if p]
122
+ return list(dict.fromkeys("/".join(parts[i:]) for i in range(len(parts))))
123
+
124
+
125
+ def _module_keys(f: str) -> list[str]:
126
+ """Import keys for a module file.
127
+
128
+ 'src/pkg/a.py' becomes '/src/pkg/a.py', '/pkg/a.py', '/pkg/a.py' -- the extra entries are
129
+ what let `crate::model` reach `src/model.rs` and `from pkg.a import x` reach `src/pkg/a.py`,
130
+ since import paths never repeat the repository's own leading directories. Package roots
131
+ (`__init__.py`, `mod.rs`) additionally answer to their bare directory name.
132
+ """
133
+ d, base = posixpath.split(f)
134
+ keys = ["/" + s for s in _suffix_paths(f)]
135
+ if base in ("__init__.py", "mod.rs") and d:
136
+ ext = posixpath.splitext(base)[1]
137
+ keys += ["/" + s + ext for s in _suffix_paths(d)]
138
+ return list(dict.fromkeys(keys))
139
+
140
+
141
+ class FileIndex:
142
+ """Lookup structures over the indexed file set, built once per resolve pass.
143
+
144
+ Import resolution is the hot path of a reindex; every lookup here is a dict hit
145
+ instead of a scan over all files. Python is resolved positionally instead (see py_lookup).
146
+ """
147
+
148
+ def __init__(self, files):
149
+ self.files = set(files)
150
+ self.rs_mod: dict[str, list[str]] = defaultdict(list)
151
+ self.go_dir: dict[str, list[str]] = defaultdict(list)
152
+ self.basename: dict[str, list[str]] = defaultdict(list)
153
+ self.stem: dict[str, list[str]] = defaultdict(list)
154
+ # conventional Python source roots, present only if this repository uses them
155
+ self.import_roots = [""] + [d for d in ("src", "lib") if any(f.startswith(d + "/") for f in self.files)]
156
+ for f in sorted(self.files):
157
+ base = posixpath.basename(f)
158
+ self.stem[posixpath.splitext(base)[0]].append(f)
159
+ self.basename[base].append(f)
160
+ self._register(f, base)
161
+
162
+ def py_lookup(self, dotted: str) -> list[str]:
163
+ """Resolve an absolute dotted module against the repository's import roots.
164
+
165
+ Python resolves absolute imports against sys.path, not against the importing file's own
166
+ directory, so this deliberately does *not* walk up from the importer: that would make
167
+ `from typing import cast` inside `pydantic/_internal/` find `pydantic/typing.py`. The
168
+ import roots are the repository root plus the conventional source roots that exist here.
169
+ """
170
+ if not dotted:
171
+ return []
172
+ rel = dotted.replace(".", "/")
173
+ for base in self.import_roots:
174
+ target = posixpath.join(base, rel) if base else rel
175
+ hit = next((target + suf for suf in (".py", "/__init__.py") if target + suf in self.files), None)
176
+ if hit:
177
+ return [hit]
178
+ return []
179
+
180
+ def _register(self, f: str, base: str):
181
+ """Register a file under every import key that can refer to it. See _module_keys."""
182
+ ext = posixpath.splitext(base)[1]
183
+ if ext == ".rs":
184
+ for k in _module_keys(f):
185
+ self.rs_mod[k].append(f)
186
+ elif ext == ".go":
187
+ for k in _suffix_paths(posixpath.dirname(f)):
188
+ self.go_dir[k].append(f)
189
+
190
+
191
+ JS_SUFFIXES = ["", ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs",
192
+ "/index.ts", "/index.tsx", "/index.js", "/index.jsx", "/index.mjs", "/index.cjs"]
193
+ CPP_EXTS = {".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".inl", ".ipp", ".tpp"}
194
+
195
+
196
+ def resolve_import(raw: str, src: str, idx: FileIndex) -> list[str]:
197
+ """Map one raw import string to the repo files it most likely refers to."""
198
+ raw = raw.strip()
199
+ if not raw:
200
+ return []
201
+ ext = posixpath.splitext(src)[1]
202
+ sdir = posixpath.dirname(src)
203
+
204
+ if ext == ".py":
205
+ dots = len(raw) - len(raw.lstrip("."))
206
+ rest = raw.lstrip(".").replace(".", "/")
207
+ if dots: # `from .mod import x`: relative to the importing package
208
+ base = sdir
209
+ for _ in range(dots - 1):
210
+ base = posixpath.dirname(base)
211
+ target = posixpath.normpath(posixpath.join(base, rest)) if rest else base
212
+ return sorted(f for s in (".py", "/__init__.py") if (f := target + s) in idx.files)
213
+ # absolute: search the import roots, not the importer's directory (see py_lookup)
214
+ return idx.py_lookup(rest)
215
+
216
+ if ext in (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"):
217
+ if raw.startswith("."):
218
+ return _js_candidates(posixpath.normpath(posixpath.join(sdir, raw)), idx)
219
+ # bare specifier: resolvable only inside this repo's own workspace layout
220
+ return _js_candidates(posixpath.normpath(raw.lstrip("/")), idx)
221
+
222
+ if ext == ".go":
223
+ segs = [s for s in raw.strip('"`').split("/") if s and s != "."]
224
+ for i in range(len(segs)): # longest matching path suffix wins
225
+ hit = idx.go_dir.get("/".join(segs[i:]))
226
+ if hit:
227
+ return sorted(hit)[:10]
228
+ return []
229
+
230
+ if ext == ".rs":
231
+ segs = [s for s in re.split(r"[:{},\s*]+", raw) if s and s not in ("crate", "use", "self", "super")]
232
+ for i in range(len(segs)):
233
+ for j in range(len(segs), i, -1):
234
+ hit = idx.rs_mod.get("/" + "/".join(segs[i:j]) + ".rs")
235
+ if hit:
236
+ return sorted(hit)
237
+ for s in segs:
238
+ hit = idx.stem.get(s)
239
+ if hit and len(hit) == 1:
240
+ return list(hit)
241
+ return []
242
+
243
+ if ext in CPP_EXTS or ext in (".cc", ".cpp"):
244
+ name = raw.strip('<>"\'')
245
+ if not name:
246
+ return []
247
+ if "/" in name: # "dir/foo.h": must match the trailing path as well
248
+ return sorted(f for f in idx.files if f.endswith("/" + name) or f == name)[:3]
249
+ base = posixpath.basename(name)
250
+ hits = [f for f in idx.basename.get(base, []) if posixpath.splitext(f)[1] in CPP_EXTS]
251
+ if not hits and "." not in name: # extensionless: `foo` -> foo.h / foo.hpp / ...
252
+ hits = [f for f in idx.files
253
+ if posixpath.splitext(f)[0].endswith("/" + name) and posixpath.splitext(f)[1] in CPP_EXTS]
254
+ same_dir = [f for f in hits if posixpath.dirname(f) == sdir]
255
+ return sorted(same_dir or hits)[:3]
256
+
257
+ return []
258
+
259
+
260
+ def _js_candidates(base: str, idx: FileIndex) -> list[str]:
261
+ for s in JS_SUFFIXES:
262
+ if base + s in idx.files:
263
+ return [base + s]
264
+ return []
265
+
266
+
267
+ class Store:
268
+ def __init__(self, path: Path):
269
+ path.parent.mkdir(parents=True, exist_ok=True)
270
+ self.db = sqlite3.connect(path, check_same_thread=False)
271
+ self.db.row_factory = sqlite3.Row
272
+ # WAL lets the long-lived server connection read while a reindex writes
273
+ self.db.execute("PRAGMA journal_mode=WAL")
274
+ self.db.execute("PRAGMA synchronous=NORMAL")
275
+ self.db.executescript(SCHEMA)
276
+ self._resolved_raws: dict[str, set[str]] = {}
277
+ self.migrated = self._migrate()
278
+ # explicit ids let add_file insert a whole file's symbols in one executemany
279
+ self._next_id = (self.db.execute("SELECT MAX(id) FROM symbols").fetchone()[0] or 0) + 1
280
+
281
+ def _migrate(self) -> bool:
282
+ """Apply outstanding schema migrations. Every step must be idempotent.
283
+
284
+ Only writes when something actually changes, and commits immediately: leaving an open
285
+ write transaction here would block the indexer's own connection for the lifetime of the
286
+ MCP server (this connection is long-lived and `check_same_thread=False`).
287
+ """
288
+ db = self.db
289
+ row = db.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
290
+ current = int(row["value"]) if row else 0
291
+ schema_changed = False
292
+ for version in sorted(MIGRATIONS):
293
+ if version <= current: # already applied
294
+ continue
295
+ if MIGRATIONS[version](db):
296
+ schema_changed = True
297
+ if current != SCHEMA_VERSION:
298
+ db.execute("INSERT INTO meta(key,value) VALUES('schema_version',?) "
299
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (str(SCHEMA_VERSION),))
300
+ if current != SCHEMA_VERSION or schema_changed:
301
+ db.commit()
302
+ # reports whether an existing schema was actually altered, not merely version-stamped
303
+ return schema_changed
304
+
305
+ def get_meta(self, key: str, default: str | None = None) -> str | None:
306
+ row = self.db.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
307
+ return row["value"] if row else default
308
+
309
+ def set_meta(self, key: str, value: str):
310
+ self.db.execute("INSERT INTO meta(key,value) VALUES(?,?) "
311
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
312
+ self.db.commit() # never leave a write transaction open on a long-lived connection
313
+
314
+ def close(self):
315
+ self.db.close()
316
+
317
+ def commit(self):
318
+ self.db.commit()
319
+
320
+ # ---- files ----
321
+ def file_hashes(self) -> dict[str, str]:
322
+ return {r["path"]: r["hash"] for r in self.db.execute("SELECT path, hash FROM files")}
323
+
324
+ def names_in(self, files) -> set[str]:
325
+ """Every symbol name defined in the given files (used to scope incremental resolution)."""
326
+ files = list(files)
327
+ if not files:
328
+ return set()
329
+ rows = self.db.execute(
330
+ f"SELECT DISTINCT name FROM symbols WHERE file IN ({_ph(files)})", files)
331
+ return {r[0] for r in rows}
332
+
333
+ def importers_of(self, files) -> set[str]:
334
+ """Files whose resolved imports point at any of `files`.
335
+
336
+ Must be called *before* deleting those files: deleting a file removes the import rows
337
+ that point at it, after which the affected importers can no longer be discovered.
338
+ """
339
+ files = list(files)
340
+ if not files:
341
+ return set()
342
+ rows = self.db.execute(f"SELECT DISTINCT file FROM imports WHERE dst IN ({_ph(files)})",
343
+ files).fetchall()
344
+ return {r[0] for r in rows}
345
+
346
+ def clear(self):
347
+ """Drop every derived row (not the embedding cache, which is keyed by content hash).
348
+ A forced rebuild is far cheaper this way than deleting file by file."""
349
+ for t in ("files", "symbols", "raw_edges", "edges", "raw_imports", "imports", "raw_keys",
350
+ "links", "file_links", "sym_vec", "sym_fts"):
351
+ self.db.execute(f"DELETE FROM {t}")
352
+
353
+ def delete_file(self, path: str):
354
+ """Remove a file and everything derived from it, so the store is never left inconsistent."""
355
+ db = self.db
356
+ ids = [r[0] for r in db.execute("SELECT id FROM symbols WHERE file=?", (path,))]
357
+ for i in range(0, len(ids), 500):
358
+ chunk = ids[i:i + 500]
359
+ key = _ph(chunk)
360
+ for table, col in (("raw_edges", "src"), ("sym_fts", "rowid"), ("raw_keys", "src"),
361
+ ("sym_vec", "sym_id")):
362
+ db.execute(f"DELETE FROM {table} WHERE {col} IN ({key})", chunk)
363
+ db.execute(f"DELETE FROM edges WHERE src IN ({key}) OR dst IN ({key})", chunk + chunk)
364
+ db.execute(f"DELETE FROM links WHERE src IN ({key}) OR dst IN ({key})", chunk + chunk)
365
+ db.execute("DELETE FROM file_links WHERE a=? OR b=?", (path, path))
366
+ db.execute("DELETE FROM imports WHERE file=? OR dst=?", (path, path))
367
+ db.execute("DELETE FROM raw_imports WHERE file=?", (path,))
368
+ db.execute("DELETE FROM symbols WHERE file=?", (path,))
369
+ db.execute("DELETE FROM files WHERE path=?", (path,))
370
+
371
+ def add_file(self, path: str, lang: str, h: str, p: Parsed, loc: int):
372
+ db = self.db
373
+ db.execute("INSERT INTO files VALUES(?,?,?,?)", (path, lang, h, loc))
374
+ idmap: dict[int, int] = {}
375
+ sym_rows, fts_rows = [], []
376
+ for s in p.syms:
377
+ sid = self._next_id
378
+ self._next_id += 1
379
+ idmap[s.id] = sid
380
+ parent = idmap.get(s.parent) if s.parent is not None else None
381
+ q = s.qualname or s.name
382
+ sym_rows.append((sid, path, s.name, q, s.kind, s.start, s.end, s.sig, s.doc, parent))
383
+ fts_rows.append((sid, s.name, split_words(s.name), q, s.doc or ""))
384
+ db.executemany("INSERT INTO symbols(id,file,name,qualname,kind,line_start,line_end,sig,doc,parent) "
385
+ "VALUES(?,?,?,?,?,?,?,?,?,?)", sym_rows)
386
+ db.executemany("INSERT INTO sym_fts(rowid,name,words,qualname,doc) VALUES(?,?,?,?,?)", fts_rows)
387
+ db.executemany("INSERT INTO raw_edges VALUES(?,?,?)",
388
+ [(idmap[src], name, kind) for src, name, kind in p.refs])
389
+ db.executemany("INSERT INTO raw_keys VALUES(?,?,?,?)",
390
+ [(idmap[src], ns, key, role) for src, ns, key, role in dict.fromkeys(p.keys)])
391
+ db.executemany("INSERT INTO raw_imports VALUES(?,?)",
392
+ [(path, raw) for raw in dict.fromkeys(p.imports)]) # de-duped, order preserved
393
+
394
+ # ---- import resolution ----
395
+ def _imported_map(self) -> dict[str, set[str]]:
396
+ imported: dict[str, set[str]] = defaultdict(set)
397
+ for f, d in self.db.execute("SELECT file, dst FROM imports"):
398
+ imported[f].add(d)
399
+ return imported
400
+
401
+ @staticmethod
402
+ def _base(raw: str) -> str:
403
+ return raw.strip().lstrip(".").split(".", 1)[0]
404
+
405
+ @staticmethod
406
+ def _leaf(raw: str) -> str:
407
+ return raw.strip().lstrip(".").replace("::", ".").rsplit("/", 1)[-1].rsplit(".", 1)[-1]
408
+
409
+ def _external_imports(self, rows) -> dict[str, set[str]]:
410
+ """Leaf names a file imports from modules that are not in the repo.
411
+
412
+ `from typing import cast` in a file that has nothing to do with SQLAlchemy must not make a
413
+ bare `cast(...)` call resolve to `ColumnElement.cast`. A name bound to a module we cannot
414
+ see is treated as living outside the graph.
415
+
416
+ A dotted raw whose *base* module did resolve is only a speculative submodule guess
417
+ (`from .util import shout` also emits `util.shout`), so it is not evidence of anything
418
+ external.
419
+ """
420
+ per_file: dict[str, list[str]] = defaultdict(list)
421
+ for r in rows:
422
+ per_file[r["file"]].append(r["raw"])
423
+ out: dict[str, set[str]] = defaultdict(set)
424
+ for file, raws in per_file.items():
425
+ known_bases = set()
426
+ for raw in raws:
427
+ if raw in self._resolved_raws.get(file, ()):
428
+ known_bases.add(self._base(raw))
429
+ for raw in raws:
430
+ if raw in self._resolved_raws.get(file, ()):
431
+ continue
432
+ if self._base(raw) in known_bases:
433
+ continue # speculative submodule of a module we do know
434
+ leaf = self._leaf(raw)
435
+ if leaf:
436
+ out[file].add(leaf)
437
+ return out
438
+
439
+ def _resolve_imports(self, dirty: set[str] | None = None) -> dict[str, set[str]]:
440
+ """(Re)resolve file-level imports. With `dirty` set, only those files and their
441
+ importers are recomputed; otherwise the whole table is rebuilt.
442
+ Returns (imported, external) maps."""
443
+ db = self.db
444
+ files = [r["path"] for r in db.execute("SELECT path FROM files")]
445
+ if dirty is None:
446
+ db.execute("DELETE FROM imports")
447
+ targets = files
448
+ else:
449
+ affected = set(dirty)
450
+ for r in db.execute(f"SELECT DISTINCT file FROM imports WHERE dst IN ({_ph(affected)})",
451
+ list(affected)).fetchall():
452
+ affected.add(r[0])
453
+ db.execute(f"DELETE FROM imports WHERE file IN ({_ph(affected)})", list(affected))
454
+ targets = [f for f in files if f in affected]
455
+ # Resolution runs over every file, but only the affected files' rows are written. The
456
+ # external-import map must cover untouched files too: edges are re-resolved for sources in
457
+ # any file, and a missing entry would turn `from typing import cast` into a phantom edge.
458
+ idx = FileIndex(files)
459
+ rows = db.execute("SELECT file, raw FROM raw_imports").fetchall()
460
+ affected_set = set(targets)
461
+ found, resolved = set(), defaultdict(set)
462
+ for r in rows:
463
+ hits = [d for d in resolve_import(r["raw"], r["file"], idx) if d != r["file"]]
464
+ if hits:
465
+ resolved[r["file"]].add(r["raw"])
466
+ if r["file"] in affected_set:
467
+ found.update((r["file"], d) for d in hits)
468
+ self._resolved_raws = resolved
469
+ external = self._external_imports(rows)
470
+ db.executemany("INSERT INTO imports VALUES(?,?)", sorted(found))
471
+ return self._imported_map(), external
472
+
473
+ # ---- edge resolution ----
474
+ def resolve(self, *, full: bool = False, dirty_files=None, dirty_names=None) -> dict:
475
+ """Resolve raw references into edges.
476
+
477
+ `full=True` (or no dirty set) rebuilds everything. Otherwise only raw edges whose
478
+ name, or whose source file, is affected are recomputed; everything else keeps its
479
+ edges. Returns a small stats dict.
480
+ """
481
+ db = self.db
482
+ incremental = not full and dirty_files is not None
483
+ imported, external = self._resolve_imports(
484
+ None if full or not incremental else set(dirty_files))
485
+
486
+ file_of, name_of, kind_of, parent_of = {}, {}, {}, {}
487
+ by_name: dict[str, list[tuple[int, str, str]]] = defaultdict(list)
488
+ for r in db.execute("SELECT id, name, file, kind, parent FROM symbols"):
489
+ file_of[r["id"]] = r["file"]
490
+ name_of[r["id"]], kind_of[r["id"]], parent_of[r["id"]] = r["name"], r["kind"], r["parent"]
491
+ if r["kind"] not in ("module", "impl", "namespace"):
492
+ by_name[r["name"]].append((r["id"], r["file"], r["kind"]))
493
+
494
+ bases: dict[str, set[str]] = defaultdict(set)
495
+ subs: dict[str, set[str]] = defaultdict(set)
496
+ for e in db.execute("SELECT src, name FROM raw_edges WHERE kind='inherits'").fetchall():
497
+ child = name_of.get(e["src"])
498
+ if child is None:
499
+ continue
500
+ bases[child].add(e["name"])
501
+ subs[e["name"]].add(child)
502
+
503
+ class_cache: dict[int, str | None] = {}
504
+
505
+ def class_of(i: int) -> str | None:
506
+ if i not in class_cache:
507
+ j, found = parent_of.get(i), None
508
+ while j is not None:
509
+ if kind_of.get(j) in CLASS_KINDS:
510
+ found = name_of.get(j)
511
+ break
512
+ j = parent_of.get(j)
513
+ class_cache[i] = found
514
+ return class_cache[i]
515
+
516
+ def resolve_one(src: int, name: str, kind: str) -> list[tuple[int, int, str, str]]:
517
+ cands = by_name.get(name)
518
+ if not cands:
519
+ return []
520
+ self_cn = class_of(src) if kind == "calls_self" else None
521
+ if kind == "calls_self" and self_cn is None:
522
+ kind = "calls_attr" # `self` outside any class: fall back to the generic attribute rule
523
+ is_attr = kind == "calls_attr"
524
+ out_kind = "calls" if kind.startswith("calls") else kind
525
+ allowed = TARGET_KINDS_INHERIT if out_kind == "inherits" else TARGET_KINDS_CALL
526
+ tier = "far" # the strongest evidence that picked the candidates, for the edge's confidence
527
+ cands = [c for c in cands if c[2] in allowed]
528
+ if not cands:
529
+ return []
530
+ sf = file_of[src]
531
+ if kind == "calls" and name in external.get(sf, ()):
532
+ return [] # bound to a module outside the repo, not to something in here
533
+ if kind == "calls" and name in PY_BUILTINS:
534
+ # a bare builtin call can only ever mean a module-level binding of that name;
535
+ # `Util.isinstance(...)` is a method and is spelled with a receiver
536
+ shadowed = [c for c in cands if c[1] == sf and c[2] != "method"]
537
+ if not shadowed:
538
+ return []
539
+ cands = shadowed
540
+ if self_cn is not None:
541
+ # `self.name()` can only mean a method of this class, its bases, or its subclasses.
542
+ # Nothing in that family => inherited from an external base (dict, Exception...) => no edge.
543
+ fam = _family(self_cn, bases, subs)
544
+ cands = [c for c in cands if class_of(c[0]) in fam]
545
+ if not cands:
546
+ return []
547
+ tier = "self"
548
+ else:
549
+ same = [c for c in cands if c[1] == sf]
550
+ if same:
551
+ cands, tier = same, "same"
552
+ else:
553
+ imp = [c for c in cands if c[1] in imported.get(sf, ())]
554
+ if imp:
555
+ cands, tier = imp, "imp"
556
+ else:
557
+ sd = posixpath.dirname(sf)
558
+ near = [c for c in cands if posixpath.dirname(c[1]) == sd]
559
+ # `x.name()` with no same-file / imported / same-dir candidate is most likely a
560
+ # call on some library or builtin type (dict.items, str.get...), so it gets no edge.
561
+ cands = near if is_attr else (near or cands)
562
+ tier = "near" if near else "far"
563
+ if not cands or len(cands) > MAX_CANDIDATES:
564
+ return []
565
+ cands = [c for c in cands if c[0] != src] # drop direct self-recursion
566
+ if kind == "mentions" and len(cands) != 1:
567
+ return [] # a word in prose or SQL names a symbol only when it is unambiguous
568
+ if len(cands) > 1:
569
+ conf = "ambiguous"
570
+ elif tier == "self" or (tier in ("same", "imp") and not is_attr):
571
+ conf = "extracted"
572
+ else:
573
+ conf = "inferred"
574
+ return [(src, c[0], out_kind, conf) for c in cands]
575
+
576
+ if incremental:
577
+ names = set(dirty_names or ())
578
+ names |= _family_closure(names, bases, subs) # a hierarchy change moves whole families
579
+ hit = db.execute(
580
+ f"SELECT DISTINCT r.src FROM raw_edges r JOIN symbols s ON s.id=r.src "
581
+ f"WHERE r.name IN ({_ph(names)}) OR s.file IN ({_ph(dirty_files)})",
582
+ list(names) + list(dirty_files)).fetchall()
583
+ srcs = {r[0] for r in hit}
584
+ # a touched source is recomputed from *all* of its raw references, not just the ones
585
+ # that matched: its edges are deleted wholesale, so anything not re-resolved would be lost
586
+ raw = []
587
+ for chunk in _chunks(sorted(srcs)):
588
+ raw += db.execute(f"SELECT src, name, kind FROM raw_edges WHERE src IN ({_ph(chunk)})",
589
+ chunk).fetchall()
590
+ db.execute(f"DELETE FROM edges WHERE src IN ({_ph(chunk)})", chunk)
591
+ else:
592
+ db.execute("DELETE FROM edges")
593
+ raw = db.execute("SELECT src, name, kind FROM raw_edges").fetchall()
594
+ srcs = set()
595
+
596
+ best: dict[tuple[int, int, str], str] = {}
597
+ for e in raw:
598
+ for src, dst, ek, conf in resolve_one(e["src"], e["name"], e["kind"]):
599
+ key = (src, dst, ek)
600
+ if key not in best or CONF_RANK[conf] > CONF_RANK[best[key]]:
601
+ best[key] = conf
602
+ db.executemany("INSERT INTO edges VALUES(?,?,?,?)", [(*k, c) for k, c in best.items()])
603
+
604
+ total = db.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
605
+ return {"raw_edges_considered": len(raw), "edges_total": total,
606
+ "edges_changed": bool(srcs) or not incremental}
607
+
608
+
609
+ def _family(cn: str, bases: dict[str, set[str]], subs: dict[str, set[str]]) -> frozenset[str]:
610
+ seen = {cn}
611
+ for graph in (bases, subs):
612
+ todo = [cn]
613
+ while todo:
614
+ for nxt in graph.get(todo.pop(), ()):
615
+ if nxt not in seen:
616
+ seen.add(nxt)
617
+ todo.append(nxt)
618
+ return frozenset(seen)
619
+
620
+
621
+ def _family_closure(names, bases: dict[str, set[str]], subs: dict[str, set[str]]) -> set[str]:
622
+ out = set()
623
+ for n in names:
624
+ out |= _family(n, bases, subs)
625
+ return out
626
+
627
+
628
+ def recompute_rank(store: Store) -> dict:
629
+ """Global PageRank over the whole graph. This is the one step that is always a full pass,
630
+ so convergence is reported rather than assumed."""
631
+ ids = [r[0] for r in store.db.execute("SELECT id FROM symbols")]
632
+ edges = [(r[0], r[1]) for r in store.db.execute("SELECT src, dst FROM edges")]
633
+ stats: dict = {}
634
+ prev = {r[0]: r[1] for r in store.db.execute("SELECT id, rank FROM symbols")}
635
+ scores = pagerank(ids, edges, stats=stats, init=prev)
636
+ store.db.executemany("UPDATE symbols SET rank=? WHERE id=?",
637
+ [(v, k) for k, v in scores.items()])
638
+ return stats