codegraph-network 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +1 -0
- codegraph/bench.py +93 -0
- codegraph/cli.py +128 -0
- codegraph/community.py +77 -0
- codegraph/docs.py +141 -0
- codegraph/embed.py +221 -0
- codegraph/indexer.py +190 -0
- codegraph/install.py +219 -0
- codegraph/keys.py +94 -0
- codegraph/links.py +220 -0
- codegraph/parser.py +459 -0
- codegraph/py.typed +0 -0
- codegraph/queries.py +657 -0
- codegraph/rank.py +121 -0
- codegraph/report.py +199 -0
- codegraph/review.py +145 -0
- codegraph/server.py +81 -0
- codegraph/store.py +638 -0
- codegraph/tools.py +99 -0
- codegraph_network-0.1.0.dist-info/METADATA +220 -0
- codegraph_network-0.1.0.dist-info/RECORD +25 -0
- codegraph_network-0.1.0.dist-info/WHEEL +5 -0
- codegraph_network-0.1.0.dist-info/entry_points.txt +2 -0
- codegraph_network-0.1.0.dist-info/licenses/LICENSE +21 -0
- codegraph_network-0.1.0.dist-info/top_level.txt +1 -0
codegraph/store.py
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
"""SQLite graph store: symbols, resolved edges, file-level imports, FTS search."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import posixpath
|
|
5
|
+
import re
|
|
6
|
+
import sqlite3
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .parser import Parsed
|
|
11
|
+
from .rank import pagerank
|
|
12
|
+
|
|
13
|
+
SCHEMA_VERSION = 3
|
|
14
|
+
|
|
15
|
+
SCHEMA = """
|
|
16
|
+
CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY, lang TEXT, hash TEXT, loc INT);
|
|
17
|
+
CREATE TABLE IF NOT EXISTS symbols(
|
|
18
|
+
id INTEGER PRIMARY KEY, file TEXT, name TEXT, qualname TEXT, kind TEXT,
|
|
19
|
+
line_start INT, line_end INT, sig TEXT, doc TEXT, parent INT, rank REAL DEFAULT 0,
|
|
20
|
+
community INT DEFAULT -1);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS raw_edges(src INT, name TEXT, kind TEXT);
|
|
22
|
+
-- conf: how the edge was resolved. extracted = unique target named by the source itself (same file,
|
|
23
|
+
-- import, self/this); inferred = name match by proximity; ambiguous = several equally plausible targets
|
|
24
|
+
CREATE TABLE IF NOT EXISTS edges(src INT, dst INT, kind TEXT, conf TEXT DEFAULT 'inferred');
|
|
25
|
+
CREATE TABLE IF NOT EXISTS raw_imports(file TEXT, raw TEXT);
|
|
26
|
+
CREATE TABLE IF NOT EXISTS imports(file TEXT, dst TEXT);
|
|
27
|
+
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
|
|
28
|
+
CREATE INDEX IF NOT EXISTS ix_sym_name ON symbols(name);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS ix_sym_file ON symbols(file);
|
|
30
|
+
CREATE INDEX IF NOT EXISTS ix_sym_qual ON symbols(qualname);
|
|
31
|
+
CREATE INDEX IF NOT EXISTS ix_e_src ON edges(src);
|
|
32
|
+
CREATE INDEX IF NOT EXISTS ix_e_dst ON edges(dst);
|
|
33
|
+
CREATE INDEX IF NOT EXISTS ix_re_src ON raw_edges(src);
|
|
34
|
+
CREATE INDEX IF NOT EXISTS ix_imp_file ON imports(file);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS ix_imp_dst ON imports(dst);
|
|
36
|
+
CREATE TABLE IF NOT EXISTS raw_keys(src INT, ns TEXT, key TEXT, role TEXT);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS ix_rk_key ON raw_keys(ns, key);
|
|
38
|
+
CREATE INDEX IF NOT EXISTS ix_rk_src ON raw_keys(src);
|
|
39
|
+
-- non-structural symbol links (similar / tests / key). Kept apart from `edges` so callers, callees,
|
|
40
|
+
-- impact and global PageRank stay purely structural.
|
|
41
|
+
CREATE TABLE IF NOT EXISTS links(src INT, dst INT, kind TEXT, weight REAL, note TEXT);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS ix_l_src ON links(src);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS ix_l_dst ON links(dst);
|
|
44
|
+
CREATE TABLE IF NOT EXISTS file_links(a TEXT, b TEXT, kind TEXT, weight REAL);
|
|
45
|
+
CREATE INDEX IF NOT EXISTS ix_fl_a ON file_links(a);
|
|
46
|
+
CREATE INDEX IF NOT EXISTS ix_fl_b ON file_links(b);
|
|
47
|
+
-- embeddings: vectors are cached by content hash, so only changed symbols are ever re-embedded
|
|
48
|
+
CREATE TABLE IF NOT EXISTS sym_vec(sym_id INTEGER PRIMARY KEY, hash TEXT);
|
|
49
|
+
CREATE TABLE IF NOT EXISTS vec_cache(hash TEXT PRIMARY KEY, vec BLOB);
|
|
50
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS sym_fts USING fts5(name, words, qualname, doc);
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _migration_1(db: sqlite3.Connection) -> bool:
|
|
55
|
+
"""Add the rank column (absent in databases written before ranking) and speed up name lookups."""
|
|
56
|
+
db.execute("CREATE INDEX IF NOT EXISTS ix_re_name ON raw_edges(name)")
|
|
57
|
+
cols = {r[1] for r in db.execute("PRAGMA table_info(symbols)")}
|
|
58
|
+
if "rank" in cols:
|
|
59
|
+
return False
|
|
60
|
+
db.execute("ALTER TABLE symbols ADD COLUMN rank REAL DEFAULT 0")
|
|
61
|
+
return True
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _migration_2(db: sqlite3.Connection) -> bool:
|
|
65
|
+
"""Links, keys and embedding tables: created by SCHEMA, which is idempotent. Nothing to alter."""
|
|
66
|
+
return False
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _migration_3(db: sqlite3.Connection) -> bool:
|
|
70
|
+
"""Edge confidence and symbol community. Old databases are re-indexed (parser version bump)."""
|
|
71
|
+
changed = False
|
|
72
|
+
for table, col, ddl in (("edges", "conf", "TEXT DEFAULT 'inferred'"),
|
|
73
|
+
("symbols", "community", "INT DEFAULT -1")):
|
|
74
|
+
if col not in {r[1] for r in db.execute(f"PRAGMA table_info({table})")}:
|
|
75
|
+
db.execute(f"ALTER TABLE {table} ADD COLUMN {col} {ddl}")
|
|
76
|
+
changed = True
|
|
77
|
+
return changed
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
MIGRATIONS = {1: _migration_1, 2: _migration_2, 3: _migration_3}
|
|
81
|
+
|
|
82
|
+
TARGET_KINDS_CALL = {"function", "method", "class", "struct", "type", "interface", "trait", "enum",
|
|
83
|
+
"table", "view", "procedure"} # SQL objects, referenced from docs and other SQL
|
|
84
|
+
TARGET_KINDS_INHERIT = {"class", "interface", "trait", "struct", "type", "enum"}
|
|
85
|
+
CLASS_KINDS = {"class", "struct", "trait", "interface", "impl"}
|
|
86
|
+
# kinds that are containers: a `function` inside one is really a method
|
|
87
|
+
CONTAINER_KINDS = {"class", "struct", "trait", "interface", "impl"}
|
|
88
|
+
CONF_RANK = {"ambiguous": 0, "inferred": 1, "extracted": 2}
|
|
89
|
+
MAX_CANDIDATES = 4 # names defined in more places than this are too ambiguous to link
|
|
90
|
+
|
|
91
|
+
# Python builtins called directly (`isinstance(x, T)`) are not calls to anything in the repo.
|
|
92
|
+
# Without this, a repo method that happens to be called `isinstance` becomes a hub with thousands
|
|
93
|
+
# of phantom in-edges and dominates the global PageRank. A same-file definition still wins, so
|
|
94
|
+
# code that deliberately shadows a builtin keeps working.
|
|
95
|
+
PY_BUILTINS = frozenset(
|
|
96
|
+
"abs all any ascii bin bool breakpoint bytearray bytes callable chr classmethod compile complex "
|
|
97
|
+
"dict dir divmod enumerate eval exec filter float format frozenset getattr globals hasattr hash "
|
|
98
|
+
"help hex id input int isinstance issubclass iter len list locals map max memoryview min next "
|
|
99
|
+
"object oct open ord pow print property range repr reversed round set setattr slice sorted "
|
|
100
|
+
"staticmethod str sum super tuple type vars zip".split()
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def split_words(name: str) -> str:
|
|
105
|
+
return re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", name).replace("_", " ")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _ph(values) -> str:
|
|
109
|
+
"""Placeholder list for an IN clause. Safe for empty input."""
|
|
110
|
+
vals = list(values)
|
|
111
|
+
return ",".join("?" * len(vals)) if vals else "NULL"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _chunks(items, n: int = 500):
|
|
115
|
+
for i in range(0, len(items), n):
|
|
116
|
+
yield items[i:i + n]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _suffix_paths(path: str) -> list[str]:
|
|
120
|
+
"""Every trailing sub-path of `path`, each exactly once: 'a/b/c' -> ['a/b/c', 'b/c', 'c']."""
|
|
121
|
+
parts = [p for p in path.split("/") if p]
|
|
122
|
+
return list(dict.fromkeys("/".join(parts[i:]) for i in range(len(parts))))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _module_keys(f: str) -> list[str]:
|
|
126
|
+
"""Import keys for a module file.
|
|
127
|
+
|
|
128
|
+
'src/pkg/a.py' becomes '/src/pkg/a.py', '/pkg/a.py', '/pkg/a.py' -- the extra entries are
|
|
129
|
+
what let `crate::model` reach `src/model.rs` and `from pkg.a import x` reach `src/pkg/a.py`,
|
|
130
|
+
since import paths never repeat the repository's own leading directories. Package roots
|
|
131
|
+
(`__init__.py`, `mod.rs`) additionally answer to their bare directory name.
|
|
132
|
+
"""
|
|
133
|
+
d, base = posixpath.split(f)
|
|
134
|
+
keys = ["/" + s for s in _suffix_paths(f)]
|
|
135
|
+
if base in ("__init__.py", "mod.rs") and d:
|
|
136
|
+
ext = posixpath.splitext(base)[1]
|
|
137
|
+
keys += ["/" + s + ext for s in _suffix_paths(d)]
|
|
138
|
+
return list(dict.fromkeys(keys))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class FileIndex:
|
|
142
|
+
"""Lookup structures over the indexed file set, built once per resolve pass.
|
|
143
|
+
|
|
144
|
+
Import resolution is the hot path of a reindex; every lookup here is a dict hit
|
|
145
|
+
instead of a scan over all files. Python is resolved positionally instead (see py_lookup).
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
def __init__(self, files):
|
|
149
|
+
self.files = set(files)
|
|
150
|
+
self.rs_mod: dict[str, list[str]] = defaultdict(list)
|
|
151
|
+
self.go_dir: dict[str, list[str]] = defaultdict(list)
|
|
152
|
+
self.basename: dict[str, list[str]] = defaultdict(list)
|
|
153
|
+
self.stem: dict[str, list[str]] = defaultdict(list)
|
|
154
|
+
# conventional Python source roots, present only if this repository uses them
|
|
155
|
+
self.import_roots = [""] + [d for d in ("src", "lib") if any(f.startswith(d + "/") for f in self.files)]
|
|
156
|
+
for f in sorted(self.files):
|
|
157
|
+
base = posixpath.basename(f)
|
|
158
|
+
self.stem[posixpath.splitext(base)[0]].append(f)
|
|
159
|
+
self.basename[base].append(f)
|
|
160
|
+
self._register(f, base)
|
|
161
|
+
|
|
162
|
+
def py_lookup(self, dotted: str) -> list[str]:
|
|
163
|
+
"""Resolve an absolute dotted module against the repository's import roots.
|
|
164
|
+
|
|
165
|
+
Python resolves absolute imports against sys.path, not against the importing file's own
|
|
166
|
+
directory, so this deliberately does *not* walk up from the importer: that would make
|
|
167
|
+
`from typing import cast` inside `pydantic/_internal/` find `pydantic/typing.py`. The
|
|
168
|
+
import roots are the repository root plus the conventional source roots that exist here.
|
|
169
|
+
"""
|
|
170
|
+
if not dotted:
|
|
171
|
+
return []
|
|
172
|
+
rel = dotted.replace(".", "/")
|
|
173
|
+
for base in self.import_roots:
|
|
174
|
+
target = posixpath.join(base, rel) if base else rel
|
|
175
|
+
hit = next((target + suf for suf in (".py", "/__init__.py") if target + suf in self.files), None)
|
|
176
|
+
if hit:
|
|
177
|
+
return [hit]
|
|
178
|
+
return []
|
|
179
|
+
|
|
180
|
+
def _register(self, f: str, base: str):
|
|
181
|
+
"""Register a file under every import key that can refer to it. See _module_keys."""
|
|
182
|
+
ext = posixpath.splitext(base)[1]
|
|
183
|
+
if ext == ".rs":
|
|
184
|
+
for k in _module_keys(f):
|
|
185
|
+
self.rs_mod[k].append(f)
|
|
186
|
+
elif ext == ".go":
|
|
187
|
+
for k in _suffix_paths(posixpath.dirname(f)):
|
|
188
|
+
self.go_dir[k].append(f)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
JS_SUFFIXES = ["", ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs",
|
|
192
|
+
"/index.ts", "/index.tsx", "/index.js", "/index.jsx", "/index.mjs", "/index.cjs"]
|
|
193
|
+
CPP_EXTS = {".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".inl", ".ipp", ".tpp"}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def resolve_import(raw: str, src: str, idx: FileIndex) -> list[str]:
|
|
197
|
+
"""Map one raw import string to the repo files it most likely refers to."""
|
|
198
|
+
raw = raw.strip()
|
|
199
|
+
if not raw:
|
|
200
|
+
return []
|
|
201
|
+
ext = posixpath.splitext(src)[1]
|
|
202
|
+
sdir = posixpath.dirname(src)
|
|
203
|
+
|
|
204
|
+
if ext == ".py":
|
|
205
|
+
dots = len(raw) - len(raw.lstrip("."))
|
|
206
|
+
rest = raw.lstrip(".").replace(".", "/")
|
|
207
|
+
if dots: # `from .mod import x`: relative to the importing package
|
|
208
|
+
base = sdir
|
|
209
|
+
for _ in range(dots - 1):
|
|
210
|
+
base = posixpath.dirname(base)
|
|
211
|
+
target = posixpath.normpath(posixpath.join(base, rest)) if rest else base
|
|
212
|
+
return sorted(f for s in (".py", "/__init__.py") if (f := target + s) in idx.files)
|
|
213
|
+
# absolute: search the import roots, not the importer's directory (see py_lookup)
|
|
214
|
+
return idx.py_lookup(rest)
|
|
215
|
+
|
|
216
|
+
if ext in (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"):
|
|
217
|
+
if raw.startswith("."):
|
|
218
|
+
return _js_candidates(posixpath.normpath(posixpath.join(sdir, raw)), idx)
|
|
219
|
+
# bare specifier: resolvable only inside this repo's own workspace layout
|
|
220
|
+
return _js_candidates(posixpath.normpath(raw.lstrip("/")), idx)
|
|
221
|
+
|
|
222
|
+
if ext == ".go":
|
|
223
|
+
segs = [s for s in raw.strip('"`').split("/") if s and s != "."]
|
|
224
|
+
for i in range(len(segs)): # longest matching path suffix wins
|
|
225
|
+
hit = idx.go_dir.get("/".join(segs[i:]))
|
|
226
|
+
if hit:
|
|
227
|
+
return sorted(hit)[:10]
|
|
228
|
+
return []
|
|
229
|
+
|
|
230
|
+
if ext == ".rs":
|
|
231
|
+
segs = [s for s in re.split(r"[:{},\s*]+", raw) if s and s not in ("crate", "use", "self", "super")]
|
|
232
|
+
for i in range(len(segs)):
|
|
233
|
+
for j in range(len(segs), i, -1):
|
|
234
|
+
hit = idx.rs_mod.get("/" + "/".join(segs[i:j]) + ".rs")
|
|
235
|
+
if hit:
|
|
236
|
+
return sorted(hit)
|
|
237
|
+
for s in segs:
|
|
238
|
+
hit = idx.stem.get(s)
|
|
239
|
+
if hit and len(hit) == 1:
|
|
240
|
+
return list(hit)
|
|
241
|
+
return []
|
|
242
|
+
|
|
243
|
+
if ext in CPP_EXTS or ext in (".cc", ".cpp"):
|
|
244
|
+
name = raw.strip('<>"\'')
|
|
245
|
+
if not name:
|
|
246
|
+
return []
|
|
247
|
+
if "/" in name: # "dir/foo.h": must match the trailing path as well
|
|
248
|
+
return sorted(f for f in idx.files if f.endswith("/" + name) or f == name)[:3]
|
|
249
|
+
base = posixpath.basename(name)
|
|
250
|
+
hits = [f for f in idx.basename.get(base, []) if posixpath.splitext(f)[1] in CPP_EXTS]
|
|
251
|
+
if not hits and "." not in name: # extensionless: `foo` -> foo.h / foo.hpp / ...
|
|
252
|
+
hits = [f for f in idx.files
|
|
253
|
+
if posixpath.splitext(f)[0].endswith("/" + name) and posixpath.splitext(f)[1] in CPP_EXTS]
|
|
254
|
+
same_dir = [f for f in hits if posixpath.dirname(f) == sdir]
|
|
255
|
+
return sorted(same_dir or hits)[:3]
|
|
256
|
+
|
|
257
|
+
return []
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _js_candidates(base: str, idx: FileIndex) -> list[str]:
|
|
261
|
+
for s in JS_SUFFIXES:
|
|
262
|
+
if base + s in idx.files:
|
|
263
|
+
return [base + s]
|
|
264
|
+
return []
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class Store:
|
|
268
|
+
def __init__(self, path: Path):
|
|
269
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
270
|
+
self.db = sqlite3.connect(path, check_same_thread=False)
|
|
271
|
+
self.db.row_factory = sqlite3.Row
|
|
272
|
+
# WAL lets the long-lived server connection read while a reindex writes
|
|
273
|
+
self.db.execute("PRAGMA journal_mode=WAL")
|
|
274
|
+
self.db.execute("PRAGMA synchronous=NORMAL")
|
|
275
|
+
self.db.executescript(SCHEMA)
|
|
276
|
+
self._resolved_raws: dict[str, set[str]] = {}
|
|
277
|
+
self.migrated = self._migrate()
|
|
278
|
+
# explicit ids let add_file insert a whole file's symbols in one executemany
|
|
279
|
+
self._next_id = (self.db.execute("SELECT MAX(id) FROM symbols").fetchone()[0] or 0) + 1
|
|
280
|
+
|
|
281
|
+
def _migrate(self) -> bool:
|
|
282
|
+
"""Apply outstanding schema migrations. Every step must be idempotent.
|
|
283
|
+
|
|
284
|
+
Only writes when something actually changes, and commits immediately: leaving an open
|
|
285
|
+
write transaction here would block the indexer's own connection for the lifetime of the
|
|
286
|
+
MCP server (this connection is long-lived and `check_same_thread=False`).
|
|
287
|
+
"""
|
|
288
|
+
db = self.db
|
|
289
|
+
row = db.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
|
|
290
|
+
current = int(row["value"]) if row else 0
|
|
291
|
+
schema_changed = False
|
|
292
|
+
for version in sorted(MIGRATIONS):
|
|
293
|
+
if version <= current: # already applied
|
|
294
|
+
continue
|
|
295
|
+
if MIGRATIONS[version](db):
|
|
296
|
+
schema_changed = True
|
|
297
|
+
if current != SCHEMA_VERSION:
|
|
298
|
+
db.execute("INSERT INTO meta(key,value) VALUES('schema_version',?) "
|
|
299
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (str(SCHEMA_VERSION),))
|
|
300
|
+
if current != SCHEMA_VERSION or schema_changed:
|
|
301
|
+
db.commit()
|
|
302
|
+
# reports whether an existing schema was actually altered, not merely version-stamped
|
|
303
|
+
return schema_changed
|
|
304
|
+
|
|
305
|
+
def get_meta(self, key: str, default: str | None = None) -> str | None:
|
|
306
|
+
row = self.db.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
|
307
|
+
return row["value"] if row else default
|
|
308
|
+
|
|
309
|
+
def set_meta(self, key: str, value: str):
|
|
310
|
+
self.db.execute("INSERT INTO meta(key,value) VALUES(?,?) "
|
|
311
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
|
312
|
+
self.db.commit() # never leave a write transaction open on a long-lived connection
|
|
313
|
+
|
|
314
|
+
def close(self):
|
|
315
|
+
self.db.close()
|
|
316
|
+
|
|
317
|
+
def commit(self):
|
|
318
|
+
self.db.commit()
|
|
319
|
+
|
|
320
|
+
# ---- files ----
|
|
321
|
+
def file_hashes(self) -> dict[str, str]:
|
|
322
|
+
return {r["path"]: r["hash"] for r in self.db.execute("SELECT path, hash FROM files")}
|
|
323
|
+
|
|
324
|
+
def names_in(self, files) -> set[str]:
|
|
325
|
+
"""Every symbol name defined in the given files (used to scope incremental resolution)."""
|
|
326
|
+
files = list(files)
|
|
327
|
+
if not files:
|
|
328
|
+
return set()
|
|
329
|
+
rows = self.db.execute(
|
|
330
|
+
f"SELECT DISTINCT name FROM symbols WHERE file IN ({_ph(files)})", files)
|
|
331
|
+
return {r[0] for r in rows}
|
|
332
|
+
|
|
333
|
+
def importers_of(self, files) -> set[str]:
|
|
334
|
+
"""Files whose resolved imports point at any of `files`.
|
|
335
|
+
|
|
336
|
+
Must be called *before* deleting those files: deleting a file removes the import rows
|
|
337
|
+
that point at it, after which the affected importers can no longer be discovered.
|
|
338
|
+
"""
|
|
339
|
+
files = list(files)
|
|
340
|
+
if not files:
|
|
341
|
+
return set()
|
|
342
|
+
rows = self.db.execute(f"SELECT DISTINCT file FROM imports WHERE dst IN ({_ph(files)})",
|
|
343
|
+
files).fetchall()
|
|
344
|
+
return {r[0] for r in rows}
|
|
345
|
+
|
|
346
|
+
def clear(self):
|
|
347
|
+
"""Drop every derived row (not the embedding cache, which is keyed by content hash).
|
|
348
|
+
A forced rebuild is far cheaper this way than deleting file by file."""
|
|
349
|
+
for t in ("files", "symbols", "raw_edges", "edges", "raw_imports", "imports", "raw_keys",
|
|
350
|
+
"links", "file_links", "sym_vec", "sym_fts"):
|
|
351
|
+
self.db.execute(f"DELETE FROM {t}")
|
|
352
|
+
|
|
353
|
+
def delete_file(self, path: str):
|
|
354
|
+
"""Remove a file and everything derived from it, so the store is never left inconsistent."""
|
|
355
|
+
db = self.db
|
|
356
|
+
ids = [r[0] for r in db.execute("SELECT id FROM symbols WHERE file=?", (path,))]
|
|
357
|
+
for i in range(0, len(ids), 500):
|
|
358
|
+
chunk = ids[i:i + 500]
|
|
359
|
+
key = _ph(chunk)
|
|
360
|
+
for table, col in (("raw_edges", "src"), ("sym_fts", "rowid"), ("raw_keys", "src"),
|
|
361
|
+
("sym_vec", "sym_id")):
|
|
362
|
+
db.execute(f"DELETE FROM {table} WHERE {col} IN ({key})", chunk)
|
|
363
|
+
db.execute(f"DELETE FROM edges WHERE src IN ({key}) OR dst IN ({key})", chunk + chunk)
|
|
364
|
+
db.execute(f"DELETE FROM links WHERE src IN ({key}) OR dst IN ({key})", chunk + chunk)
|
|
365
|
+
db.execute("DELETE FROM file_links WHERE a=? OR b=?", (path, path))
|
|
366
|
+
db.execute("DELETE FROM imports WHERE file=? OR dst=?", (path, path))
|
|
367
|
+
db.execute("DELETE FROM raw_imports WHERE file=?", (path,))
|
|
368
|
+
db.execute("DELETE FROM symbols WHERE file=?", (path,))
|
|
369
|
+
db.execute("DELETE FROM files WHERE path=?", (path,))
|
|
370
|
+
|
|
371
|
+
def add_file(self, path: str, lang: str, h: str, p: Parsed, loc: int):
|
|
372
|
+
db = self.db
|
|
373
|
+
db.execute("INSERT INTO files VALUES(?,?,?,?)", (path, lang, h, loc))
|
|
374
|
+
idmap: dict[int, int] = {}
|
|
375
|
+
sym_rows, fts_rows = [], []
|
|
376
|
+
for s in p.syms:
|
|
377
|
+
sid = self._next_id
|
|
378
|
+
self._next_id += 1
|
|
379
|
+
idmap[s.id] = sid
|
|
380
|
+
parent = idmap.get(s.parent) if s.parent is not None else None
|
|
381
|
+
q = s.qualname or s.name
|
|
382
|
+
sym_rows.append((sid, path, s.name, q, s.kind, s.start, s.end, s.sig, s.doc, parent))
|
|
383
|
+
fts_rows.append((sid, s.name, split_words(s.name), q, s.doc or ""))
|
|
384
|
+
db.executemany("INSERT INTO symbols(id,file,name,qualname,kind,line_start,line_end,sig,doc,parent) "
|
|
385
|
+
"VALUES(?,?,?,?,?,?,?,?,?,?)", sym_rows)
|
|
386
|
+
db.executemany("INSERT INTO sym_fts(rowid,name,words,qualname,doc) VALUES(?,?,?,?,?)", fts_rows)
|
|
387
|
+
db.executemany("INSERT INTO raw_edges VALUES(?,?,?)",
|
|
388
|
+
[(idmap[src], name, kind) for src, name, kind in p.refs])
|
|
389
|
+
db.executemany("INSERT INTO raw_keys VALUES(?,?,?,?)",
|
|
390
|
+
[(idmap[src], ns, key, role) for src, ns, key, role in dict.fromkeys(p.keys)])
|
|
391
|
+
db.executemany("INSERT INTO raw_imports VALUES(?,?)",
|
|
392
|
+
[(path, raw) for raw in dict.fromkeys(p.imports)]) # de-duped, order preserved
|
|
393
|
+
|
|
394
|
+
# ---- import resolution ----
|
|
395
|
+
def _imported_map(self) -> dict[str, set[str]]:
|
|
396
|
+
imported: dict[str, set[str]] = defaultdict(set)
|
|
397
|
+
for f, d in self.db.execute("SELECT file, dst FROM imports"):
|
|
398
|
+
imported[f].add(d)
|
|
399
|
+
return imported
|
|
400
|
+
|
|
401
|
+
@staticmethod
|
|
402
|
+
def _base(raw: str) -> str:
|
|
403
|
+
return raw.strip().lstrip(".").split(".", 1)[0]
|
|
404
|
+
|
|
405
|
+
@staticmethod
|
|
406
|
+
def _leaf(raw: str) -> str:
|
|
407
|
+
return raw.strip().lstrip(".").replace("::", ".").rsplit("/", 1)[-1].rsplit(".", 1)[-1]
|
|
408
|
+
|
|
409
|
+
def _external_imports(self, rows) -> dict[str, set[str]]:
|
|
410
|
+
"""Leaf names a file imports from modules that are not in the repo.
|
|
411
|
+
|
|
412
|
+
`from typing import cast` in a file that has nothing to do with SQLAlchemy must not make a
|
|
413
|
+
bare `cast(...)` call resolve to `ColumnElement.cast`. A name bound to a module we cannot
|
|
414
|
+
see is treated as living outside the graph.
|
|
415
|
+
|
|
416
|
+
A dotted raw whose *base* module did resolve is only a speculative submodule guess
|
|
417
|
+
(`from .util import shout` also emits `util.shout`), so it is not evidence of anything
|
|
418
|
+
external.
|
|
419
|
+
"""
|
|
420
|
+
per_file: dict[str, list[str]] = defaultdict(list)
|
|
421
|
+
for r in rows:
|
|
422
|
+
per_file[r["file"]].append(r["raw"])
|
|
423
|
+
out: dict[str, set[str]] = defaultdict(set)
|
|
424
|
+
for file, raws in per_file.items():
|
|
425
|
+
known_bases = set()
|
|
426
|
+
for raw in raws:
|
|
427
|
+
if raw in self._resolved_raws.get(file, ()):
|
|
428
|
+
known_bases.add(self._base(raw))
|
|
429
|
+
for raw in raws:
|
|
430
|
+
if raw in self._resolved_raws.get(file, ()):
|
|
431
|
+
continue
|
|
432
|
+
if self._base(raw) in known_bases:
|
|
433
|
+
continue # speculative submodule of a module we do know
|
|
434
|
+
leaf = self._leaf(raw)
|
|
435
|
+
if leaf:
|
|
436
|
+
out[file].add(leaf)
|
|
437
|
+
return out
|
|
438
|
+
|
|
439
|
+
def _resolve_imports(self, dirty: set[str] | None = None) -> dict[str, set[str]]:
|
|
440
|
+
"""(Re)resolve file-level imports. With `dirty` set, only those files and their
|
|
441
|
+
importers are recomputed; otherwise the whole table is rebuilt.
|
|
442
|
+
Returns (imported, external) maps."""
|
|
443
|
+
db = self.db
|
|
444
|
+
files = [r["path"] for r in db.execute("SELECT path FROM files")]
|
|
445
|
+
if dirty is None:
|
|
446
|
+
db.execute("DELETE FROM imports")
|
|
447
|
+
targets = files
|
|
448
|
+
else:
|
|
449
|
+
affected = set(dirty)
|
|
450
|
+
for r in db.execute(f"SELECT DISTINCT file FROM imports WHERE dst IN ({_ph(affected)})",
|
|
451
|
+
list(affected)).fetchall():
|
|
452
|
+
affected.add(r[0])
|
|
453
|
+
db.execute(f"DELETE FROM imports WHERE file IN ({_ph(affected)})", list(affected))
|
|
454
|
+
targets = [f for f in files if f in affected]
|
|
455
|
+
# Resolution runs over every file, but only the affected files' rows are written. The
|
|
456
|
+
# external-import map must cover untouched files too: edges are re-resolved for sources in
|
|
457
|
+
# any file, and a missing entry would turn `from typing import cast` into a phantom edge.
|
|
458
|
+
idx = FileIndex(files)
|
|
459
|
+
rows = db.execute("SELECT file, raw FROM raw_imports").fetchall()
|
|
460
|
+
affected_set = set(targets)
|
|
461
|
+
found, resolved = set(), defaultdict(set)
|
|
462
|
+
for r in rows:
|
|
463
|
+
hits = [d for d in resolve_import(r["raw"], r["file"], idx) if d != r["file"]]
|
|
464
|
+
if hits:
|
|
465
|
+
resolved[r["file"]].add(r["raw"])
|
|
466
|
+
if r["file"] in affected_set:
|
|
467
|
+
found.update((r["file"], d) for d in hits)
|
|
468
|
+
self._resolved_raws = resolved
|
|
469
|
+
external = self._external_imports(rows)
|
|
470
|
+
db.executemany("INSERT INTO imports VALUES(?,?)", sorted(found))
|
|
471
|
+
return self._imported_map(), external
|
|
472
|
+
|
|
473
|
+
# ---- edge resolution ----
|
|
474
|
+
def resolve(self, *, full: bool = False, dirty_files=None, dirty_names=None) -> dict:
|
|
475
|
+
"""Resolve raw references into edges.
|
|
476
|
+
|
|
477
|
+
`full=True` (or no dirty set) rebuilds everything. Otherwise only raw edges whose
|
|
478
|
+
name, or whose source file, is affected are recomputed; everything else keeps its
|
|
479
|
+
edges. Returns a small stats dict.
|
|
480
|
+
"""
|
|
481
|
+
db = self.db
|
|
482
|
+
incremental = not full and dirty_files is not None
|
|
483
|
+
imported, external = self._resolve_imports(
|
|
484
|
+
None if full or not incremental else set(dirty_files))
|
|
485
|
+
|
|
486
|
+
file_of, name_of, kind_of, parent_of = {}, {}, {}, {}
|
|
487
|
+
by_name: dict[str, list[tuple[int, str, str]]] = defaultdict(list)
|
|
488
|
+
for r in db.execute("SELECT id, name, file, kind, parent FROM symbols"):
|
|
489
|
+
file_of[r["id"]] = r["file"]
|
|
490
|
+
name_of[r["id"]], kind_of[r["id"]], parent_of[r["id"]] = r["name"], r["kind"], r["parent"]
|
|
491
|
+
if r["kind"] not in ("module", "impl", "namespace"):
|
|
492
|
+
by_name[r["name"]].append((r["id"], r["file"], r["kind"]))
|
|
493
|
+
|
|
494
|
+
bases: dict[str, set[str]] = defaultdict(set)
|
|
495
|
+
subs: dict[str, set[str]] = defaultdict(set)
|
|
496
|
+
for e in db.execute("SELECT src, name FROM raw_edges WHERE kind='inherits'").fetchall():
|
|
497
|
+
child = name_of.get(e["src"])
|
|
498
|
+
if child is None:
|
|
499
|
+
continue
|
|
500
|
+
bases[child].add(e["name"])
|
|
501
|
+
subs[e["name"]].add(child)
|
|
502
|
+
|
|
503
|
+
class_cache: dict[int, str | None] = {}
|
|
504
|
+
|
|
505
|
+
def class_of(i: int) -> str | None:
|
|
506
|
+
if i not in class_cache:
|
|
507
|
+
j, found = parent_of.get(i), None
|
|
508
|
+
while j is not None:
|
|
509
|
+
if kind_of.get(j) in CLASS_KINDS:
|
|
510
|
+
found = name_of.get(j)
|
|
511
|
+
break
|
|
512
|
+
j = parent_of.get(j)
|
|
513
|
+
class_cache[i] = found
|
|
514
|
+
return class_cache[i]
|
|
515
|
+
|
|
516
|
+
def resolve_one(src: int, name: str, kind: str) -> list[tuple[int, int, str, str]]:
|
|
517
|
+
cands = by_name.get(name)
|
|
518
|
+
if not cands:
|
|
519
|
+
return []
|
|
520
|
+
self_cn = class_of(src) if kind == "calls_self" else None
|
|
521
|
+
if kind == "calls_self" and self_cn is None:
|
|
522
|
+
kind = "calls_attr" # `self` outside any class: fall back to the generic attribute rule
|
|
523
|
+
is_attr = kind == "calls_attr"
|
|
524
|
+
out_kind = "calls" if kind.startswith("calls") else kind
|
|
525
|
+
allowed = TARGET_KINDS_INHERIT if out_kind == "inherits" else TARGET_KINDS_CALL
|
|
526
|
+
tier = "far" # the strongest evidence that picked the candidates, for the edge's confidence
|
|
527
|
+
cands = [c for c in cands if c[2] in allowed]
|
|
528
|
+
if not cands:
|
|
529
|
+
return []
|
|
530
|
+
sf = file_of[src]
|
|
531
|
+
if kind == "calls" and name in external.get(sf, ()):
|
|
532
|
+
return [] # bound to a module outside the repo, not to something in here
|
|
533
|
+
if kind == "calls" and name in PY_BUILTINS:
|
|
534
|
+
# a bare builtin call can only ever mean a module-level binding of that name;
|
|
535
|
+
# `Util.isinstance(...)` is a method and is spelled with a receiver
|
|
536
|
+
shadowed = [c for c in cands if c[1] == sf and c[2] != "method"]
|
|
537
|
+
if not shadowed:
|
|
538
|
+
return []
|
|
539
|
+
cands = shadowed
|
|
540
|
+
if self_cn is not None:
|
|
541
|
+
# `self.name()` can only mean a method of this class, its bases, or its subclasses.
|
|
542
|
+
# Nothing in that family => inherited from an external base (dict, Exception...) => no edge.
|
|
543
|
+
fam = _family(self_cn, bases, subs)
|
|
544
|
+
cands = [c for c in cands if class_of(c[0]) in fam]
|
|
545
|
+
if not cands:
|
|
546
|
+
return []
|
|
547
|
+
tier = "self"
|
|
548
|
+
else:
|
|
549
|
+
same = [c for c in cands if c[1] == sf]
|
|
550
|
+
if same:
|
|
551
|
+
cands, tier = same, "same"
|
|
552
|
+
else:
|
|
553
|
+
imp = [c for c in cands if c[1] in imported.get(sf, ())]
|
|
554
|
+
if imp:
|
|
555
|
+
cands, tier = imp, "imp"
|
|
556
|
+
else:
|
|
557
|
+
sd = posixpath.dirname(sf)
|
|
558
|
+
near = [c for c in cands if posixpath.dirname(c[1]) == sd]
|
|
559
|
+
# `x.name()` with no same-file / imported / same-dir candidate is most likely a
|
|
560
|
+
# call on some library or builtin type (dict.items, str.get...), so it gets no edge.
|
|
561
|
+
cands = near if is_attr else (near or cands)
|
|
562
|
+
tier = "near" if near else "far"
|
|
563
|
+
if not cands or len(cands) > MAX_CANDIDATES:
|
|
564
|
+
return []
|
|
565
|
+
cands = [c for c in cands if c[0] != src] # drop direct self-recursion
|
|
566
|
+
if kind == "mentions" and len(cands) != 1:
|
|
567
|
+
return [] # a word in prose or SQL names a symbol only when it is unambiguous
|
|
568
|
+
if len(cands) > 1:
|
|
569
|
+
conf = "ambiguous"
|
|
570
|
+
elif tier == "self" or (tier in ("same", "imp") and not is_attr):
|
|
571
|
+
conf = "extracted"
|
|
572
|
+
else:
|
|
573
|
+
conf = "inferred"
|
|
574
|
+
return [(src, c[0], out_kind, conf) for c in cands]
|
|
575
|
+
|
|
576
|
+
if incremental:
|
|
577
|
+
names = set(dirty_names or ())
|
|
578
|
+
names |= _family_closure(names, bases, subs) # a hierarchy change moves whole families
|
|
579
|
+
hit = db.execute(
|
|
580
|
+
f"SELECT DISTINCT r.src FROM raw_edges r JOIN symbols s ON s.id=r.src "
|
|
581
|
+
f"WHERE r.name IN ({_ph(names)}) OR s.file IN ({_ph(dirty_files)})",
|
|
582
|
+
list(names) + list(dirty_files)).fetchall()
|
|
583
|
+
srcs = {r[0] for r in hit}
|
|
584
|
+
# a touched source is recomputed from *all* of its raw references, not just the ones
|
|
585
|
+
# that matched: its edges are deleted wholesale, so anything not re-resolved would be lost
|
|
586
|
+
raw = []
|
|
587
|
+
for chunk in _chunks(sorted(srcs)):
|
|
588
|
+
raw += db.execute(f"SELECT src, name, kind FROM raw_edges WHERE src IN ({_ph(chunk)})",
|
|
589
|
+
chunk).fetchall()
|
|
590
|
+
db.execute(f"DELETE FROM edges WHERE src IN ({_ph(chunk)})", chunk)
|
|
591
|
+
else:
|
|
592
|
+
db.execute("DELETE FROM edges")
|
|
593
|
+
raw = db.execute("SELECT src, name, kind FROM raw_edges").fetchall()
|
|
594
|
+
srcs = set()
|
|
595
|
+
|
|
596
|
+
best: dict[tuple[int, int, str], str] = {}
|
|
597
|
+
for e in raw:
|
|
598
|
+
for src, dst, ek, conf in resolve_one(e["src"], e["name"], e["kind"]):
|
|
599
|
+
key = (src, dst, ek)
|
|
600
|
+
if key not in best or CONF_RANK[conf] > CONF_RANK[best[key]]:
|
|
601
|
+
best[key] = conf
|
|
602
|
+
db.executemany("INSERT INTO edges VALUES(?,?,?,?)", [(*k, c) for k, c in best.items()])
|
|
603
|
+
|
|
604
|
+
total = db.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
|
|
605
|
+
return {"raw_edges_considered": len(raw), "edges_total": total,
|
|
606
|
+
"edges_changed": bool(srcs) or not incremental}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _family(cn: str, bases: dict[str, set[str]], subs: dict[str, set[str]]) -> frozenset[str]:
|
|
610
|
+
seen = {cn}
|
|
611
|
+
for graph in (bases, subs):
|
|
612
|
+
todo = [cn]
|
|
613
|
+
while todo:
|
|
614
|
+
for nxt in graph.get(todo.pop(), ()):
|
|
615
|
+
if nxt not in seen:
|
|
616
|
+
seen.add(nxt)
|
|
617
|
+
todo.append(nxt)
|
|
618
|
+
return frozenset(seen)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _family_closure(names, bases: dict[str, set[str]], subs: dict[str, set[str]]) -> set[str]:
|
|
622
|
+
out = set()
|
|
623
|
+
for n in names:
|
|
624
|
+
out |= _family(n, bases, subs)
|
|
625
|
+
return out
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def recompute_rank(store: Store) -> dict:
|
|
629
|
+
"""Global PageRank over the whole graph. This is the one step that is always a full pass,
|
|
630
|
+
so convergence is reported rather than assumed."""
|
|
631
|
+
ids = [r[0] for r in store.db.execute("SELECT id FROM symbols")]
|
|
632
|
+
edges = [(r[0], r[1]) for r in store.db.execute("SELECT src, dst FROM edges")]
|
|
633
|
+
stats: dict = {}
|
|
634
|
+
prev = {r[0]: r[1] for r in store.db.execute("SELECT id, rank FROM symbols")}
|
|
635
|
+
scores = pagerank(ids, edges, stats=stats, init=prev)
|
|
636
|
+
store.db.executemany("UPDATE symbols SET rank=? WHERE id=?",
|
|
637
|
+
[(v, k) for k, v in scores.items()])
|
|
638
|
+
return stats
|