ph-code-graph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ph_code_graph/__init__.py +825 -0
- ph_code_graph/_extract.py +574 -0
- ph_code_graph/_store.py +638 -0
- ph_code_graph/bundle.yaml +21 -0
- ph_code_graph/py.typed +0 -0
- ph_code_graph/skills/code-graph/SKILL.md +66 -0
- ph_code_graph-0.1.0.dist-info/METADATA +267 -0
- ph_code_graph-0.1.0.dist-info/RECORD +11 -0
- ph_code_graph-0.1.0.dist-info/WHEEL +4 -0
- ph_code_graph-0.1.0.dist-info/entry_points.txt +5 -0
- ph_code_graph-0.1.0.dist-info/licenses/LICENSE +21 -0
ph_code_graph/_store.py
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
"""The graph on disk: four tables, an FTS index, and one recursive query.
|
|
2
|
+
|
|
3
|
+
## Why the standard library and not `pyturso`
|
|
4
|
+
|
|
5
|
+
pH's session log runs on turso, so this is the odd one out and the reason is
|
|
6
|
+
measured. Against a real 40 MB CodeGraph database (11 396 nodes, 36 830 edges),
|
|
7
|
+
`pyturso` served indexed lookups, joins and aggregates correctly — and then:
|
|
8
|
+
|
|
9
|
+
* **FTS5 is absent.** A `CREATE VIRTUAL TABLE … USING fts5` is invisible to it,
|
|
10
|
+
shadow tables included. That is `search`.
|
|
11
|
+
* **`Recursive CTEs are not yet supported`.** That is `impact`.
|
|
12
|
+
* **`json_extract` silently returns NULL** where SQLite returns the value —
|
|
13
|
+
measured `0.95` against `None` on the same rows. A wrong answer with no error
|
|
14
|
+
is worse than a missing feature, and it is what makes the first two look like
|
|
15
|
+
gaps rather than the same class of problem.
|
|
16
|
+
|
|
17
|
+
Two of the five things this store does are exactly the two turso cannot do, so
|
|
18
|
+
it uses `sqlite3` from the standard library — SQLite 3.45, FTS5 compiled in, no
|
|
19
|
+
dependency to add. If turso grows both, this is one import.
|
|
20
|
+
|
|
21
|
+
## The schema, and what it deliberately is not
|
|
22
|
+
|
|
23
|
+
`files` · `symbols` · `refs` · `imports`, plus `symbols_fts`. A **name-based**
|
|
24
|
+
graph: a reference records the name it used, and `callers`/`callees` join on
|
|
25
|
+
that name. It is not a resolved graph — two `register` methods in two classes
|
|
26
|
+
are one name here — and every query that could be ambiguous reports how many
|
|
27
|
+
definitions the name has, so the caller can see it rather than being quietly
|
|
28
|
+
given one of them.
|
|
29
|
+
|
|
30
|
+
That is the honest ceiling of this approach, and the reason is worth stating: a
|
|
31
|
+
*resolved* graph is import-graph plus scope plus type inference per language, and
|
|
32
|
+
in the tool this package replaces that was 29 708 lines of it. Name-based
|
|
33
|
+
answers most of what an agent asks ("who calls this", "what does this touch")
|
|
34
|
+
and says so where it cannot.
|
|
35
|
+
|
|
36
|
+
## Incremental by content, not by clock
|
|
37
|
+
|
|
38
|
+
A file is re-extracted when its **sha256 changes**, never on an mtime — a
|
|
39
|
+
checkout, a rebase or a `touch` moves mtimes without moving content, and
|
|
40
|
+
re-indexing a repository because git changed some timestamps is the kind of cost
|
|
41
|
+
nobody attributes correctly. Re-running the indexer over an unchanged tree
|
|
42
|
+
therefore reads and hashes, and writes nothing.
|
|
43
|
+
|
|
44
|
+
@module ph_code_graph._store
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import hashlib
|
|
50
|
+
import logging
|
|
51
|
+
import sqlite3
|
|
52
|
+
import time
|
|
53
|
+
from collections.abc import Iterator, Sequence
|
|
54
|
+
from contextlib import closing, contextmanager
|
|
55
|
+
from dataclasses import dataclass
|
|
56
|
+
from pathlib import Path
|
|
57
|
+
from typing import Any
|
|
58
|
+
|
|
59
|
+
from ._extract import Extraction, owners
|
|
60
|
+
|
|
61
|
+
__all__ = ["EMPTY_STATS", "CodeGraphStore", "Hit", "IndexVersion", "SymbolRow", "digest_of"]
|
|
62
|
+
|
|
63
|
+
log = logging.getLogger("ph_code_graph.store")
|
|
64
|
+
|
|
65
|
+
SCHEMA_VERSION = 1
|
|
66
|
+
"""The baseline. Nothing has shipped, so there is no second shape in the world
|
|
67
|
+
and no migration to write — `SCHEMA` is simply what a version-1 index is.
|
|
68
|
+
|
|
69
|
+
Read and refused by `prepare()` even so, because that is the half that was
|
|
70
|
+
missing rather than a courtesy: `meta` was written on every open and read
|
|
71
|
+
nowhere, so it was state that could only ever be wrong. The first bump that
|
|
72
|
+
costs anybody a rebuild is the first one that will be believed."""
|
|
73
|
+
|
|
74
|
+
EMPTY_STATS: dict[str, Any] = {"files": 0, "symbols": 0, "refs": 0, "languages": []}
|
|
75
|
+
"""What an absent or schemaless index reports. One literal, two callers."""
|
|
76
|
+
|
|
77
|
+
SCHEMA = """
|
|
78
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
79
|
+
|
|
80
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
81
|
+
path TEXT PRIMARY KEY,
|
|
82
|
+
language TEXT NOT NULL,
|
|
83
|
+
digest TEXT NOT NULL,
|
|
84
|
+
vcs_id TEXT NOT NULL DEFAULT '',
|
|
85
|
+
lines INTEGER NOT NULL,
|
|
86
|
+
indexed_at INTEGER NOT NULL
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
90
|
+
id INTEGER PRIMARY KEY,
|
|
91
|
+
path TEXT NOT NULL,
|
|
92
|
+
name TEXT NOT NULL,
|
|
93
|
+
kind TEXT NOT NULL,
|
|
94
|
+
start_line INTEGER NOT NULL,
|
|
95
|
+
end_line INTEGER NOT NULL,
|
|
96
|
+
doc TEXT
|
|
97
|
+
);
|
|
98
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
|
|
99
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_path ON symbols(path, start_line);
|
|
100
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kind ON symbols(kind);
|
|
101
|
+
|
|
102
|
+
CREATE TABLE IF NOT EXISTS refs (
|
|
103
|
+
id INTEGER PRIMARY KEY,
|
|
104
|
+
path TEXT NOT NULL,
|
|
105
|
+
name TEXT NOT NULL,
|
|
106
|
+
kind TEXT NOT NULL,
|
|
107
|
+
line INTEGER NOT NULL,
|
|
108
|
+
from_symbol INTEGER
|
|
109
|
+
);
|
|
110
|
+
CREATE INDEX IF NOT EXISTS idx_refs_name ON refs(name);
|
|
111
|
+
CREATE INDEX IF NOT EXISTS idx_refs_from ON refs(from_symbol);
|
|
112
|
+
CREATE INDEX IF NOT EXISTS idx_refs_path ON refs(path);
|
|
113
|
+
|
|
114
|
+
CREATE TABLE IF NOT EXISTS imports (
|
|
115
|
+
path TEXT NOT NULL,
|
|
116
|
+
source TEXT NOT NULL
|
|
117
|
+
);
|
|
118
|
+
CREATE INDEX IF NOT EXISTS idx_imports_path ON imports(path);
|
|
119
|
+
|
|
120
|
+
-- `content=''` — an external-content table would have to be kept in step with
|
|
121
|
+
-- `symbols` by triggers, and this index is rebuilt wholesale for a file at a
|
|
122
|
+
-- time by the indexer anyway. Contentless is smaller and has one writer.
|
|
123
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
|
|
124
|
+
name, doc, path, content=''
|
|
125
|
+
);
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class IndexVersion(RuntimeError):
|
|
130
|
+
"""The database was written by a schema this build does not read.
|
|
131
|
+
|
|
132
|
+
Its own type because the only fix is a human decision — delete and rebuild
|
|
133
|
+
— exactly as `ph_text_index._store.IndexMismatch` is for the same reason.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _version_of(connection: sqlite3.Connection) -> int | None:
|
|
138
|
+
"""The stamped schema version, or `None` for a database that has none yet."""
|
|
139
|
+
try:
|
|
140
|
+
row = connection.execute("SELECT value FROM meta WHERE key = 'schema'").fetchone()
|
|
141
|
+
except sqlite3.OperationalError:
|
|
142
|
+
return None
|
|
143
|
+
try:
|
|
144
|
+
return int(row["value"]) if row is not None else None
|
|
145
|
+
except (TypeError, ValueError):
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def digest_of(text: str) -> str:
|
|
150
|
+
"""The content key. See the module docstring on incrementality."""
|
|
151
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass(frozen=True, slots=True)
|
|
155
|
+
class SymbolRow:
|
|
156
|
+
"""One definition, as the model reads it."""
|
|
157
|
+
|
|
158
|
+
name: str
|
|
159
|
+
kind: str
|
|
160
|
+
path: str
|
|
161
|
+
start_line: int
|
|
162
|
+
end_line: int
|
|
163
|
+
doc: str | None = None
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def lines(self) -> int:
|
|
167
|
+
"""How many lines the definition spans.
|
|
168
|
+
|
|
169
|
+
Derived rather than stored. It was a field fed by an `AS span` clause in
|
|
170
|
+
six separate SELECTs, so a seventh query that forgot the alias failed at
|
|
171
|
+
row-mapping time rather than at the query — and it was a third value
|
|
172
|
+
that had to agree with two others.
|
|
173
|
+
"""
|
|
174
|
+
return self.end_line - self.start_line + 1
|
|
175
|
+
|
|
176
|
+
def as_value(self) -> dict[str, Any]:
|
|
177
|
+
return {
|
|
178
|
+
"name": self.name,
|
|
179
|
+
"kind": self.kind,
|
|
180
|
+
"path": self.path,
|
|
181
|
+
"start_line": self.start_line,
|
|
182
|
+
"end_line": self.end_line,
|
|
183
|
+
"doc": self.doc,
|
|
184
|
+
"lines": self.lines,
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass(frozen=True, slots=True)
|
|
189
|
+
class Hit:
|
|
190
|
+
"""An edge: a symbol at the far end, and where the reference itself is.
|
|
191
|
+
|
|
192
|
+
**`path` and `ref_path` are different files and both matter.** For `callers`
|
|
193
|
+
they coincide — the calling symbol contains the call. For `callees` they do
|
|
194
|
+
not: the symbol is the callee's *definition*, while the reference is a line
|
|
195
|
+
in the caller. Rendering one against the other printed `target.py:941` for a
|
|
196
|
+
line that lives in `context.py`, which is a pointer to nothing.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
symbol: SymbolRow
|
|
200
|
+
line: int
|
|
201
|
+
"""Where the reference is written — the line to open."""
|
|
202
|
+
ref_path: str
|
|
203
|
+
"""Which file that line is in. See the class docstring."""
|
|
204
|
+
via: str
|
|
205
|
+
"""The referenced name, which for a name-based graph is the edge's label."""
|
|
206
|
+
|
|
207
|
+
def as_value(self) -> dict[str, Any]:
|
|
208
|
+
return {
|
|
209
|
+
**self.symbol.as_value(),
|
|
210
|
+
"ref_line": self.line,
|
|
211
|
+
"ref_path": self.ref_path,
|
|
212
|
+
"via": self.via,
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@dataclass(slots=True)
|
|
217
|
+
class CodeGraphStore:
|
|
218
|
+
"""The index. **Blocking**; the seam calls it in a worker thread."""
|
|
219
|
+
|
|
220
|
+
path: Path
|
|
221
|
+
|
|
222
|
+
# ------------------------------------------------------------ lifecycle ----
|
|
223
|
+
|
|
224
|
+
@contextmanager
|
|
225
|
+
def _open(self) -> Iterator[sqlite3.Connection]:
|
|
226
|
+
"""A connection with the pragmas this index wants, closed on the way out.
|
|
227
|
+
|
|
228
|
+
Opened per call rather than held: `sqlite3` connections are not safe to
|
|
229
|
+
share across threads, and every caller here arrives on whichever worker
|
|
230
|
+
`to_thread` picked. WAL so a read during a write does not block, and
|
|
231
|
+
`foreign_keys` off because the cascades are done in Python — a file's
|
|
232
|
+
rows are deleted by path, which is one statement per table and clearer
|
|
233
|
+
than a trigger nobody sees.
|
|
234
|
+
|
|
235
|
+
**It creates nothing.** It used to take `write=True` and then replay
|
|
236
|
+
`SCHEMA` plus a `meta` upsert on every call — and `put` is per file, so a
|
|
237
|
+
20 000-file index ran 20 000 no-op schema scripts and 20 000 durable
|
|
238
|
+
commits of a row nobody read. Measured at 1.8 ms per file, 24 % of all
|
|
239
|
+
write time, and 70x on a synthetic run where WAL checkpointed per file.
|
|
240
|
+
`prepare()` is the one writer of the schema, and the indexer already
|
|
241
|
+
calls it once before the loop.
|
|
242
|
+
"""
|
|
243
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
244
|
+
with closing(sqlite3.connect(self.path, isolation_level=None)) as connection:
|
|
245
|
+
connection.execute("PRAGMA journal_mode = WAL")
|
|
246
|
+
connection.execute("PRAGMA synchronous = NORMAL")
|
|
247
|
+
connection.row_factory = sqlite3.Row
|
|
248
|
+
yield connection
|
|
249
|
+
|
|
250
|
+
@contextmanager
|
|
251
|
+
def _writing(self) -> Iterator[sqlite3.Connection]:
|
|
252
|
+
"""One connection with an actual transaction around it.
|
|
253
|
+
|
|
254
|
+
**`with connection:` is not one here**, which is the trap this exists to
|
|
255
|
+
close. `_open` asks for `isolation_level=None` so that a read costs no
|
|
256
|
+
implicit transaction and `PRAGMA journal_mode` can take effect at all —
|
|
257
|
+
and in autocommit the connection's own context manager has no
|
|
258
|
+
transaction to commit, so each of a file's ~60 statements was its own
|
|
259
|
+
durable commit. Measured over 137 real files: **8.19 ms per file as it
|
|
260
|
+
was, 3.42 ms with this**, for the atomicity `put`'s docstring already
|
|
261
|
+
claimed.
|
|
262
|
+
|
|
263
|
+
`BEGIN IMMEDIATE` rather than a deferred begin: this only ever wraps
|
|
264
|
+
writes, and taking the write lock up front turns a later
|
|
265
|
+
mid-transaction contention into an honest wait at the start.
|
|
266
|
+
|
|
267
|
+
Used by the single-statement writer too, which does not need it: what
|
|
268
|
+
that one needed was to stop *reading* as though `, connection:` were a
|
|
269
|
+
transaction, since that is the whole mistake.
|
|
270
|
+
"""
|
|
271
|
+
with self._open() as connection:
|
|
272
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
273
|
+
try:
|
|
274
|
+
yield connection
|
|
275
|
+
except BaseException:
|
|
276
|
+
connection.execute("ROLLBACK")
|
|
277
|
+
raise
|
|
278
|
+
connection.execute("COMMIT")
|
|
279
|
+
|
|
280
|
+
def prepare(self) -> None:
|
|
281
|
+
"""Create the schema and stamp its version. Idempotent.
|
|
282
|
+
|
|
283
|
+
:raises IndexVersion: when the file on disk was written by a schema this
|
|
284
|
+
build does not read. Checked rather than assumed, which is the half
|
|
285
|
+
that was missing: `meta` was written on every open and read nowhere,
|
|
286
|
+
so it was state that could only ever be wrong. The sibling package's
|
|
287
|
+
sidecar `format` is read and refused, and this is that.
|
|
288
|
+
"""
|
|
289
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
290
|
+
with closing(sqlite3.connect(self.path, isolation_level=None)) as connection:
|
|
291
|
+
connection.row_factory = sqlite3.Row
|
|
292
|
+
connection.execute("PRAGMA journal_mode = WAL")
|
|
293
|
+
found = _version_of(connection)
|
|
294
|
+
if found is not None and found != SCHEMA_VERSION:
|
|
295
|
+
raise IndexVersion(
|
|
296
|
+
f"{self.path} is schema {found} and this build reads {SCHEMA_VERSION}; "
|
|
297
|
+
"delete the file to rebuild"
|
|
298
|
+
)
|
|
299
|
+
connection.executescript(SCHEMA)
|
|
300
|
+
connection.execute(
|
|
301
|
+
"INSERT OR REPLACE INTO meta(key, value) VALUES ('schema', ?)",
|
|
302
|
+
(str(SCHEMA_VERSION),),
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
def exists(self) -> bool:
|
|
306
|
+
return self.path.exists()
|
|
307
|
+
|
|
308
|
+
# -------------------------------------------------------------- indexing ----
|
|
309
|
+
|
|
310
|
+
def known(self) -> dict[str, tuple[str, str]]:
|
|
311
|
+
"""`{path: (digest, vcs_id)}` for everything already indexed.
|
|
312
|
+
|
|
313
|
+
The whole map in one read, because the indexer's next act is to compare
|
|
314
|
+
every candidate against it — a `SELECT` per file would be one statement
|
|
315
|
+
per file to answer a question one statement answers.
|
|
316
|
+
|
|
317
|
+
Both ids, because the indexer asks two questions per file and they have
|
|
318
|
+
different answers: `vcs_id` decides whether the file need be *opened*,
|
|
319
|
+
and `digest` decides whether what was opened is different.
|
|
320
|
+
"""
|
|
321
|
+
if not self.exists():
|
|
322
|
+
return {}
|
|
323
|
+
with self._open() as connection:
|
|
324
|
+
try:
|
|
325
|
+
rows = connection.execute("SELECT path, digest, vcs_id FROM files").fetchall()
|
|
326
|
+
except sqlite3.OperationalError:
|
|
327
|
+
# No schema yet: an empty index and an absent one are the same
|
|
328
|
+
# thing to a caller, and the indexer is about to create it.
|
|
329
|
+
return {}
|
|
330
|
+
return {row["path"]: (row["digest"], row["vcs_id"]) for row in rows}
|
|
331
|
+
|
|
332
|
+
def token(self) -> str:
|
|
333
|
+
"""The version-control token stored at the last index. `""` if none.
|
|
334
|
+
|
|
335
|
+
jj's working-copy id, which its one-call diff needs as a starting point.
|
|
336
|
+
Per index rather than per file, because it describes the *tree*."""
|
|
337
|
+
if not self.exists():
|
|
338
|
+
return ""
|
|
339
|
+
with self._open() as connection:
|
|
340
|
+
try:
|
|
341
|
+
row = connection.execute("SELECT value FROM meta WHERE key = 'vcs'").fetchone()
|
|
342
|
+
except sqlite3.OperationalError:
|
|
343
|
+
return ""
|
|
344
|
+
return str(row["value"]) if row is not None else ""
|
|
345
|
+
|
|
346
|
+
def remember(self, token: str) -> None:
|
|
347
|
+
"""Store the token for the next run to diff from."""
|
|
348
|
+
with self._writing() as connection:
|
|
349
|
+
connection.execute(
|
|
350
|
+
"INSERT OR REPLACE INTO meta(key, value) VALUES ('vcs', ?)", (token,)
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
def forget(self, paths: Sequence[str]) -> int:
|
|
354
|
+
"""Drop every row belonging to `paths`. Returns how many files went."""
|
|
355
|
+
if not paths:
|
|
356
|
+
return 0
|
|
357
|
+
with self._writing() as connection:
|
|
358
|
+
gone = self._forget(connection, paths)
|
|
359
|
+
return gone
|
|
360
|
+
|
|
361
|
+
def _forget(self, connection: sqlite3.Connection, paths: Sequence[str]) -> int:
|
|
362
|
+
gone = 0
|
|
363
|
+
for path in paths:
|
|
364
|
+
# The FTS rows first, by the id they were inserted under: a
|
|
365
|
+
# contentless table cannot be asked which rows belong to a path, so
|
|
366
|
+
# the ids come from `symbols` while it still has them.
|
|
367
|
+
ids = [
|
|
368
|
+
row["id"]
|
|
369
|
+
for row in connection.execute("SELECT id FROM symbols WHERE path = ?", (path,))
|
|
370
|
+
]
|
|
371
|
+
connection.executemany(
|
|
372
|
+
"INSERT INTO symbols_fts(symbols_fts, rowid, name, doc, path) "
|
|
373
|
+
"VALUES ('delete', ?, '', '', '')",
|
|
374
|
+
[(one,) for one in ids],
|
|
375
|
+
)
|
|
376
|
+
connection.execute("DELETE FROM symbols WHERE path = ?", (path,))
|
|
377
|
+
connection.execute("DELETE FROM refs WHERE path = ?", (path,))
|
|
378
|
+
connection.execute("DELETE FROM imports WHERE path = ?", (path,))
|
|
379
|
+
gone += connection.execute("DELETE FROM files WHERE path = ?", (path,)).rowcount
|
|
380
|
+
return gone
|
|
381
|
+
|
|
382
|
+
def put(self, path: str, digest: str, extraction: Extraction, vcs_id: str = "") -> int:
|
|
383
|
+
"""Replace one file's rows with `extraction`. Returns symbols written.
|
|
384
|
+
|
|
385
|
+
Replace, not merge: a file is the unit of extraction, so the old rows are
|
|
386
|
+
deleted and the new ones written in one transaction. A crash mid-write
|
|
387
|
+
therefore leaves the file absent from `files`, which the indexer treats
|
|
388
|
+
as "not indexed" and does again — rather than leaving half its symbols
|
|
389
|
+
behind, which nothing would ever notice.
|
|
390
|
+
"""
|
|
391
|
+
with self._writing() as connection:
|
|
392
|
+
self._forget(connection, [path])
|
|
393
|
+
connection.execute(
|
|
394
|
+
"INSERT INTO files(path, language, digest, vcs_id, lines, indexed_at) "
|
|
395
|
+
"VALUES (?, ?, ?, ?, ?, ?)",
|
|
396
|
+
(
|
|
397
|
+
path,
|
|
398
|
+
extraction.language,
|
|
399
|
+
digest,
|
|
400
|
+
vcs_id,
|
|
401
|
+
extraction.lines,
|
|
402
|
+
int(time.time()),
|
|
403
|
+
),
|
|
404
|
+
)
|
|
405
|
+
by_line: dict[int, int] = {}
|
|
406
|
+
for definition in extraction.definitions:
|
|
407
|
+
cursor = connection.execute(
|
|
408
|
+
"INSERT INTO symbols(path, name, kind, start_line, end_line, doc) "
|
|
409
|
+
"VALUES (?, ?, ?, ?, ?, ?)",
|
|
410
|
+
(
|
|
411
|
+
path,
|
|
412
|
+
definition.name,
|
|
413
|
+
definition.kind,
|
|
414
|
+
definition.start_line,
|
|
415
|
+
definition.end_line,
|
|
416
|
+
definition.doc,
|
|
417
|
+
),
|
|
418
|
+
)
|
|
419
|
+
identifier = int(cursor.lastrowid or 0)
|
|
420
|
+
by_line[definition.start_line] = identifier
|
|
421
|
+
connection.execute(
|
|
422
|
+
"INSERT INTO symbols_fts(rowid, name, doc, path) VALUES (?, ?, ?, ?)",
|
|
423
|
+
(identifier, definition.name, definition.doc or "", path),
|
|
424
|
+
)
|
|
425
|
+
# One `{line: tightest definition}` table for the whole file, rather
|
|
426
|
+
# than a scan of every definition per reference: that was
|
|
427
|
+
# O(definitions x references), which a generated file with 1 500
|
|
428
|
+
# definitions and 3 000 references turned into 58 ms — as much as its
|
|
429
|
+
# entire parse.
|
|
430
|
+
#
|
|
431
|
+
# Resolved here rather than by a later pass either way: the
|
|
432
|
+
# containing definition is known while the file's own definitions are
|
|
433
|
+
# in hand, and a join that recomputed containment per query would be
|
|
434
|
+
# a range scan over every symbol in the file.
|
|
435
|
+
owner = owners(extraction.definitions)
|
|
436
|
+
connection.executemany(
|
|
437
|
+
"INSERT INTO refs(path, name, kind, line, from_symbol) VALUES (?, ?, ?, ?, ?)",
|
|
438
|
+
[
|
|
439
|
+
(
|
|
440
|
+
path,
|
|
441
|
+
reference.name,
|
|
442
|
+
reference.kind,
|
|
443
|
+
reference.line,
|
|
444
|
+
by_line.get(found.start_line) if found is not None else None,
|
|
445
|
+
)
|
|
446
|
+
for reference in extraction.references
|
|
447
|
+
for found in (owner.get(reference.line),)
|
|
448
|
+
],
|
|
449
|
+
)
|
|
450
|
+
connection.executemany(
|
|
451
|
+
"INSERT INTO imports(path, source) VALUES (?, ?)",
|
|
452
|
+
[(path, one) for one in extraction.imports],
|
|
453
|
+
)
|
|
454
|
+
return len(extraction.definitions)
|
|
455
|
+
|
|
456
|
+
# --------------------------------------------------------------- queries ----
|
|
457
|
+
|
|
458
|
+
def search(self, query: str, limit: int) -> list[SymbolRow]:
|
|
459
|
+
"""Names and docstrings, by relevance. FTS5, which is why not turso."""
|
|
460
|
+
with self._open() as connection:
|
|
461
|
+
rows = connection.execute(
|
|
462
|
+
"""SELECT s.*
|
|
463
|
+
FROM symbols_fts f JOIN symbols s ON s.id = f.rowid
|
|
464
|
+
WHERE symbols_fts MATCH ?
|
|
465
|
+
ORDER BY bm25(symbols_fts, 10.0, 1.0, 0.0) LIMIT ?""",
|
|
466
|
+
(query, limit),
|
|
467
|
+
).fetchall()
|
|
468
|
+
return [_row(one) for one in rows]
|
|
469
|
+
|
|
470
|
+
def define(self, name: str, limit: int) -> list[SymbolRow]:
|
|
471
|
+
"""Every definition of an exact name."""
|
|
472
|
+
with self._open() as connection:
|
|
473
|
+
rows = connection.execute(
|
|
474
|
+
"""SELECT * FROM symbols
|
|
475
|
+
WHERE name = ? ORDER BY path, start_line LIMIT ?""",
|
|
476
|
+
(name, limit),
|
|
477
|
+
).fetchall()
|
|
478
|
+
return [_row(one) for one in rows]
|
|
479
|
+
|
|
480
|
+
def callers(self, name: str, limit: int) -> list[Hit]:
|
|
481
|
+
"""Symbols containing a reference to `name`, and the line of each."""
|
|
482
|
+
with self._open() as connection:
|
|
483
|
+
rows = connection.execute(
|
|
484
|
+
"""SELECT s.*,
|
|
485
|
+
r.line AS ref_line, r.path AS ref_path
|
|
486
|
+
FROM refs r JOIN symbols s ON s.id = r.from_symbol
|
|
487
|
+
WHERE r.name = ? ORDER BY s.path, r.line LIMIT ?""",
|
|
488
|
+
(name, limit),
|
|
489
|
+
).fetchall()
|
|
490
|
+
return [
|
|
491
|
+
Hit(symbol=_row(one), line=one["ref_line"], ref_path=one["ref_path"], via=name)
|
|
492
|
+
for one in rows
|
|
493
|
+
]
|
|
494
|
+
|
|
495
|
+
def callees(self, name: str, limit: int) -> list[Hit]:
|
|
496
|
+
"""Definitions of the names referenced from inside `name`."""
|
|
497
|
+
with self._open() as connection:
|
|
498
|
+
rows = connection.execute(
|
|
499
|
+
"""SELECT t.*,
|
|
500
|
+
r.line AS ref_line, r.path AS ref_path, r.name AS via
|
|
501
|
+
FROM symbols s
|
|
502
|
+
JOIN refs r ON r.from_symbol = s.id
|
|
503
|
+
JOIN symbols t ON t.name = r.name
|
|
504
|
+
WHERE s.name = ? ORDER BY r.line, t.path LIMIT ?""",
|
|
505
|
+
(name, limit),
|
|
506
|
+
).fetchall()
|
|
507
|
+
return [
|
|
508
|
+
Hit(
|
|
509
|
+
symbol=_row(one),
|
|
510
|
+
line=one["ref_line"],
|
|
511
|
+
ref_path=one["ref_path"],
|
|
512
|
+
via=one["via"],
|
|
513
|
+
)
|
|
514
|
+
for one in rows
|
|
515
|
+
]
|
|
516
|
+
|
|
517
|
+
def impact(self, name: str, distance: int, limit: int) -> list[tuple[int, SymbolRow]]:
|
|
518
|
+
"""Transitive callers, `1..distance` hops out, with the hop each is at.
|
|
519
|
+
|
|
520
|
+
**The recursive CTE that turso cannot run.** One statement rather than a
|
|
521
|
+
BFS of N queries per ring — the difference is not style: a Python loop
|
|
522
|
+
would issue a query per frontier symbol, and this is the query an agent
|
|
523
|
+
asks when it wants to know what a change breaks, over a graph where the
|
|
524
|
+
interesting names have hundreds of callers.
|
|
525
|
+
|
|
526
|
+
`MIN(depth)` because a symbol reachable at two distances is at the
|
|
527
|
+
nearer one, which is what makes the rings a budget rather than a
|
|
528
|
+
multiset — the same rule the old row's hand-written walk had.
|
|
529
|
+
"""
|
|
530
|
+
with self._open() as connection:
|
|
531
|
+
rows = connection.execute(
|
|
532
|
+
"""WITH RECURSIVE reach(id, depth) AS (
|
|
533
|
+
SELECT id, 0 FROM symbols WHERE name = ?
|
|
534
|
+
UNION
|
|
535
|
+
SELECT s.id, r.depth + 1
|
|
536
|
+
FROM reach r
|
|
537
|
+
JOIN symbols origin ON origin.id = r.id
|
|
538
|
+
JOIN refs f ON f.name = origin.name
|
|
539
|
+
JOIN symbols s ON s.id = f.from_symbol
|
|
540
|
+
WHERE r.depth < ?
|
|
541
|
+
)
|
|
542
|
+
SELECT MIN(depth) AS depth, s.*
|
|
543
|
+
FROM reach JOIN symbols s ON s.id = reach.id
|
|
544
|
+
WHERE depth > 0
|
|
545
|
+
GROUP BY s.id ORDER BY depth, s.path, s.start_line LIMIT ?""",
|
|
546
|
+
(name, distance, limit),
|
|
547
|
+
).fetchall()
|
|
548
|
+
return [(one["depth"], _row(one)) for one in rows]
|
|
549
|
+
|
|
550
|
+
def entities(
|
|
551
|
+
self, prefix: str | None, kind: str | None, limit: int, offset: int
|
|
552
|
+
) -> tuple[list[SymbolRow], int]:
|
|
553
|
+
"""The biggest definitions first — "what is worth opening" in one list."""
|
|
554
|
+
where = ["1 = 1"]
|
|
555
|
+
args: list[Any] = []
|
|
556
|
+
if prefix:
|
|
557
|
+
root = prefix.rstrip("/")
|
|
558
|
+
where.append("(path = ? OR path LIKE ? || '/%')")
|
|
559
|
+
args.extend([root, root])
|
|
560
|
+
if kind and kind != "any":
|
|
561
|
+
where.append("kind = ?")
|
|
562
|
+
args.append(kind)
|
|
563
|
+
clause = " AND ".join(where)
|
|
564
|
+
with self._open() as connection:
|
|
565
|
+
total = int(
|
|
566
|
+
connection.execute(f"SELECT COUNT(*) FROM symbols WHERE {clause}", args).fetchone()[
|
|
567
|
+
0
|
|
568
|
+
]
|
|
569
|
+
)
|
|
570
|
+
rows = connection.execute(
|
|
571
|
+
f"""SELECT * FROM symbols
|
|
572
|
+
WHERE {clause}
|
|
573
|
+
ORDER BY (end_line - start_line + 1) DESC, path, start_line
|
|
574
|
+
LIMIT ? OFFSET ?""",
|
|
575
|
+
[*args, limit, offset],
|
|
576
|
+
).fetchall()
|
|
577
|
+
return [_row(one) for one in rows], total
|
|
578
|
+
|
|
579
|
+
def file_count(self) -> int:
|
|
580
|
+
"""How many files are indexed. **The only number a query header needs.**
|
|
581
|
+
|
|
582
|
+
Its own method because `stats()` also counts `symbols` and `refs` and
|
|
583
|
+
groups `files` by language — three full scans a query was paying per
|
|
584
|
+
call to fill one field. Measured on a 2 000-file index: 1.97 ms against
|
|
585
|
+
0.22 ms, and it grows with the corpus without bound.
|
|
586
|
+
"""
|
|
587
|
+
if not self.exists():
|
|
588
|
+
return 0
|
|
589
|
+
with self._open() as connection:
|
|
590
|
+
try:
|
|
591
|
+
return int(connection.execute("SELECT COUNT(*) FROM files").fetchone()[0])
|
|
592
|
+
except sqlite3.OperationalError:
|
|
593
|
+
return 0
|
|
594
|
+
|
|
595
|
+
def definition_count(self, name: str) -> int:
|
|
596
|
+
"""How many places define `name` — the ambiguity a name-based graph owes.
|
|
597
|
+
|
|
598
|
+
`COUNT(*)` rather than `len(define(name, 100))`, which materialised up to
|
|
599
|
+
a hundred rows and then reported the *cap* as the count for anything
|
|
600
|
+
past it.
|
|
601
|
+
"""
|
|
602
|
+
with self._open() as connection:
|
|
603
|
+
return int(
|
|
604
|
+
connection.execute(
|
|
605
|
+
"SELECT COUNT(*) FROM symbols WHERE name = ?", (name,)
|
|
606
|
+
).fetchone()[0]
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
def stats(self) -> dict[str, Any]:
|
|
610
|
+
"""What `phern doctor` and every result's header report."""
|
|
611
|
+
if not self.exists():
|
|
612
|
+
return dict(EMPTY_STATS)
|
|
613
|
+
with self._open() as connection:
|
|
614
|
+
try:
|
|
615
|
+
files, symbols, refs = connection.execute(
|
|
616
|
+
"SELECT (SELECT COUNT(*) FROM files), (SELECT COUNT(*) FROM symbols), "
|
|
617
|
+
"(SELECT COUNT(*) FROM refs)"
|
|
618
|
+
).fetchone()
|
|
619
|
+
languages = [
|
|
620
|
+
row["language"]
|
|
621
|
+
for row in connection.execute(
|
|
622
|
+
"SELECT language FROM files GROUP BY language ORDER BY COUNT(*) DESC"
|
|
623
|
+
)
|
|
624
|
+
]
|
|
625
|
+
except sqlite3.OperationalError:
|
|
626
|
+
return dict(EMPTY_STATS)
|
|
627
|
+
return {"files": files, "symbols": symbols, "refs": refs, "languages": languages}
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _row(row: sqlite3.Row) -> SymbolRow:
|
|
631
|
+
return SymbolRow(
|
|
632
|
+
name=row["name"],
|
|
633
|
+
kind=row["kind"],
|
|
634
|
+
path=row["path"],
|
|
635
|
+
start_line=row["start_line"],
|
|
636
|
+
end_line=row["end_line"],
|
|
637
|
+
doc=row["doc"],
|
|
638
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# ph-code-graph: `code_index` and `code_graph`, over `ph-base`.
|
|
2
|
+
#
|
|
3
|
+
# A bundle rather than a row a profile spells by hand, and the reason is
|
|
4
|
+
# `available_profiles()`: a profile's composability is decided by whether its
|
|
5
|
+
# *bundles* resolve, so a profile that layers this one is correctly hidden on an
|
|
6
|
+
# install without this distribution — and the refusal names the package to
|
|
7
|
+
# install. A row named directly in a profile document gets no such check: the
|
|
8
|
+
# profile would be offered and then fail at mount, which is precisely what
|
|
9
|
+
# `ph_app.profiles` says two predicates for one question always produces.
|
|
10
|
+
#
|
|
11
|
+
# One row. `ph-code-graph` exists to register two tools, and they arrive
|
|
12
|
+
# together because `code_graph` without `code_index` is a query over an index
|
|
13
|
+
# nothing built.
|
|
14
|
+
|
|
15
|
+
# Enabled, unlike `tool-todo` and `rlm-context-loader` in their bundles. Those
|
|
16
|
+
# are off because layering their bundle for something *else* — offload,
|
|
17
|
+
# compaction — must not also hand the model a tool. Nothing else is in here:
|
|
18
|
+
# layering this bundle has exactly one meaning, so a row that stood down would
|
|
19
|
+
# make `--profile rlm-indexed` a no-op with a comment explaining why.
|
|
20
|
+
- id: code-graph
|
|
21
|
+
name: code-graph
|
ph_code_graph/py.typed
ADDED
|
File without changes
|