ph-code-graph 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ph_code_graph-0.2.0/.gitignore +31 -0
- ph_code_graph-0.2.0/LICENSE +21 -0
- ph_code_graph-0.2.0/PKG-INFO +267 -0
- ph_code_graph-0.2.0/README.md +239 -0
- ph_code_graph-0.2.0/pyproject.toml +50 -0
- ph_code_graph-0.2.0/src/ph_code_graph/__init__.py +825 -0
- ph_code_graph-0.2.0/src/ph_code_graph/_extract.py +574 -0
- ph_code_graph-0.2.0/src/ph_code_graph/_store.py +638 -0
- ph_code_graph-0.2.0/src/ph_code_graph/bundle.yaml +21 -0
- ph_code_graph-0.2.0/src/ph_code_graph/py.typed +0 -0
- ph_code_graph-0.2.0/src/ph_code_graph/skills/code-graph/SKILL.md +66 -0
- ph_code_graph-0.2.0/tests/test_code_graph.py +1193 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Build and environment
|
|
2
|
+
.venv/
|
|
3
|
+
dist/
|
|
4
|
+
build/
|
|
5
|
+
*.egg-info/
|
|
6
|
+
__pycache__/
|
|
7
|
+
*.py[cod]
|
|
8
|
+
jjt/
|
|
9
|
+
w2/
|
|
10
|
+
|
|
11
|
+
# Tooling caches
|
|
12
|
+
.pytest_cache/
|
|
13
|
+
.mypy_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.coverage
|
|
16
|
+
# Written by the guest's subprocess collectors and combined at the end of the
|
|
17
|
+
# run; see `conftest.GUEST_COVERAGE_RC`.
|
|
18
|
+
.coverage-guest*
|
|
19
|
+
htmlcov/
|
|
20
|
+
# Dropped at the repo root by pytest-textual-snapshot when a snapshot test
|
|
21
|
+
# fails. The reference snapshots under `__snapshots__/` are the committed
|
|
22
|
+
# expectation; this is the diff viewer for a run that did not match one.
|
|
23
|
+
snapshot_report.html
|
|
24
|
+
|
|
25
|
+
# Reference checkouts of the upstream projects this port reads from. Vendored
|
|
26
|
+
# locally so the plans' citations are verifiable; never part of this repo.
|
|
27
|
+
sources/
|
|
28
|
+
|
|
29
|
+
# Local scratch
|
|
30
|
+
.ph/
|
|
31
|
+
*.local.yaml
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Charles Tabor
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ph-code-graph
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: pH plugin: `code_index` and `code_graph`, a Python-native tree-sitter code graph with line-accurate answers.
|
|
5
|
+
Project-URL: Homepage, https://github.com/chastabor/pH
|
|
6
|
+
Project-URL: Repository, https://github.com/chastabor/pH
|
|
7
|
+
Project-URL: Documentation, https://github.com/chastabor/pH/blob/main/docs/README.md
|
|
8
|
+
Project-URL: Issues, https://github.com/chastabor/pH/issues
|
|
9
|
+
Author: Charles Tabor
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent,code-graph,code-search,llm,tree-sitter
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Software Development
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.12
|
|
24
|
+
Requires-Dist: ph-core==0.2.0
|
|
25
|
+
Requires-Dist: tree-sitter-language-pack>=1.16
|
|
26
|
+
Requires-Dist: tree-sitter>=0.25
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# ph-code-graph
|
|
30
|
+
|
|
31
|
+
*`code_index` and `code_graph`: ask a codebase about its own shape, and get back
|
|
32
|
+
a `path:start-end` you can `read`.*
|
|
33
|
+
|
|
34
|
+
One row, `code-graph`, registering two tools. No Node, no Rust toolchain, no
|
|
35
|
+
submodule — tree-sitter through a Python wheel, the graph in stdlib `sqlite3`.
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
phern --patch '{insert: [{id: code-graph, name: code-graph}]}' --profile llama \
|
|
39
|
+
-p "index packages/ph-core, then tell me what breaks if I change claim_slot"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## The two tools
|
|
43
|
+
|
|
44
|
+
**`code_index(paths, glob?, forget?)`** parses each file through `ctx.fs` and
|
|
45
|
+
records what it defines and references.
|
|
46
|
+
|
|
47
|
+
**Nothing watches the filesystem.** There is no daemon and no file watcher: the
|
|
48
|
+
index changes when `code_index` runs, and at no other moment. So after your own
|
|
49
|
+
edits, run it again — which is cheap by design, and cheap enough to be a habit.
|
|
50
|
+
|
|
51
|
+
**Incremental in two layers**, and the order matters:
|
|
52
|
+
|
|
53
|
+
1. *Does this file need opening?* Asked of **git or jj**, which already hold a
|
|
54
|
+
content-addressed tree — `ph.seams.changes`. A file the version control
|
|
55
|
+
vouches for is never read. Measured on `packages/ph-core/src/ph`: a re-index
|
|
56
|
+
of 137 unchanged files went from 0.10 s to **0.03 s**, with zero reads and
|
|
57
|
+
zero hashes, on a `tree_state` costing 7 ms for the whole 509-file repo.
|
|
58
|
+
2. *Is what I opened different?* Asked of the file's **sha256**, exactly as
|
|
59
|
+
before. Never an mtime — a checkout or a rebase moves timestamps without
|
|
60
|
+
moving content, and re-indexing because git touched them is a cost nobody
|
|
61
|
+
attributes correctly.
|
|
62
|
+
|
|
63
|
+
Layer 1 only decides what to open, so the guarantee layer 2 makes is untouched:
|
|
64
|
+
a skip is always justified by somebody's content hash, never by a clock. A tree
|
|
65
|
+
with no version control loses layer 1 and nothing else.
|
|
66
|
+
|
|
67
|
+
**`code_graph(mode, query?, …)`** answers six questions:
|
|
68
|
+
|
|
69
|
+
| `mode` | question | what comes back |
|
|
70
|
+
|---|---|---|
|
|
71
|
+
| `search` | "something about workspace tiers" | symbols ranked by FTS5 over name + docstring |
|
|
72
|
+
| `define` | "where exactly is `claim_slot`" | every definition of that name |
|
|
73
|
+
| `callers` | "what calls this" | the calling symbol **and the calling line** |
|
|
74
|
+
| `callees` | "what does this call" | the definitions it reaches |
|
|
75
|
+
| `impact` | "what breaks if I change this" | transitive callers, ring by ring |
|
|
76
|
+
| `entities` | "what is big here" | definitions by span, biggest first |
|
|
77
|
+
|
|
78
|
+
Every result carries `path:start-line-end-line`. That is the point of the
|
|
79
|
+
package: the model finds what matters in one call, then `read`s exactly that.
|
|
80
|
+
|
|
81
|
+
## Using it from the RLM
|
|
82
|
+
|
|
83
|
+
Nothing to do. Under Code Mode the model is handed one callable and a generated
|
|
84
|
+
SDK listing, and *every* registered tool is in that listing — so these arrive as
|
|
85
|
+
`await tools.code_index(...)` and `await tools.code_graph(...)` with no Code
|
|
86
|
+
Mode work in this package at all. The `rlm-indexed` profile is `rlm-stable` plus
|
|
87
|
+
this bundle and `ph-text-index`'s:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
phern --profile rlm-indexed --provider llama --model <model> --mode tui
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This package registers a `ph.bundles` entry point, which is what lets `ph-app`
|
|
94
|
+
compose that profile without depending on this distribution — and what makes an
|
|
95
|
+
install missing it see no `rlm-indexed` rather than one that fails at mount.
|
|
96
|
+
|
|
97
|
+
## It is a name-based graph, and it says so
|
|
98
|
+
|
|
99
|
+
A reference records the *name* it used, and `callers`/`callees` join on that
|
|
100
|
+
name. Two `register` methods in two classes are one name to this index.
|
|
101
|
+
|
|
102
|
+
Every answer that could be ambiguous carries `definitions` — how many places
|
|
103
|
+
that name is defined — and the rendered text says so outright when it is above
|
|
104
|
+
one, so the model sees the ambiguity instead of being handed one of the
|
|
105
|
+
possibilities. `mode=define` is how it disambiguates.
|
|
106
|
+
|
|
107
|
+
Resolving properly means an import graph, scope, and per-language type
|
|
108
|
+
inference. In the Rust/Node CodeGraph that is 29,708 lines of TypeScript, and
|
|
109
|
+
this package deliberately did not port it. Name-based matching answers most of
|
|
110
|
+
what an agent actually asks and reports where it cannot — which is a better
|
|
111
|
+
trade than a resolved graph for one language.
|
|
112
|
+
|
|
113
|
+
## Why Python-native, and not a wrapper
|
|
114
|
+
|
|
115
|
+
The obvious plan was to submodule the Rust/Node CodeGraph and wrap its kernel
|
|
116
|
+
with maturin. The investigation said no, for three reasons in increasing order
|
|
117
|
+
of importance:
|
|
118
|
+
|
|
119
|
+
1. **`codegraph-kernel` is a `#[napi]` crate** — `crate-type = ["cdylib"]`
|
|
120
|
+
against `napi`/`napi-derive`. maturin builds PyO3/cffi extensions; wrapping
|
|
121
|
+
it means forking the crate to rewrite its boundary.
|
|
122
|
+
2. **It is only an extractor.** Its whole export surface is `extract_file`,
|
|
123
|
+
`contract_info`, `grammar_info` and two `cfnptr` helpers — "tree-sitter
|
|
124
|
+
parse+extract with one JS boundary crossing per file", in its own words. It
|
|
125
|
+
also has a complete TS/WASM fallback, so it is an accelerator, not the core.
|
|
126
|
+
3. **The intelligence is TypeScript**: `resolution/` 29,708 lines, `extraction/`
|
|
127
|
+
25,612, `mcp/` 13,940, `graph/` 6,023, `db/` 5,475 — 112,612 against the
|
|
128
|
+
kernel's 25,242. Wrapping the kernel gets a parser and leaves the product
|
|
129
|
+
behind.
|
|
130
|
+
|
|
131
|
+
And the parser is the part Python already has. `tree-sitter-language-pack` 1.16
|
|
132
|
+
ships **26 languages compiled into a 3.7 MB wheel, working offline** (python,
|
|
133
|
+
ts/tsx/js, rust, go, java, csharp, c, cpp, ruby, php, swift, kotlin, scala,
|
|
134
|
+
dart, lua, r, and more), with 371 available and the long tail fetched on
|
|
135
|
+
demand — which this row never triggers implicitly.
|
|
136
|
+
|
|
137
|
+
## Why `sqlite3` and not `pyturso`
|
|
138
|
+
|
|
139
|
+
pH's session log runs on turso, so this is the exception and the reason is
|
|
140
|
+
measured. Against a real 40 MB CodeGraph database (11,396 nodes, 36,830 edges),
|
|
141
|
+
`pyturso` served indexed lookups, joins and aggregates correctly, and then:
|
|
142
|
+
|
|
143
|
+
- **FTS5 is absent** — a `CREATE VIRTUAL TABLE … USING fts5` is invisible to
|
|
144
|
+
it, shadow tables included. That is `search`.
|
|
145
|
+
- **`Recursive CTEs are not yet supported`** — that is `impact`.
|
|
146
|
+
- **`json_extract` silently returns NULL** where SQLite returns the value:
|
|
147
|
+
`0.95` against `None`, same rows, same expression. A wrong answer with no
|
|
148
|
+
error is worse than a missing feature.
|
|
149
|
+
|
|
150
|
+
Two of the six modes here are exactly the two turso cannot serve. Stdlib
|
|
151
|
+
`sqlite3` (SQLite 3.45, FTS5 compiled in) costs no dependency and does all six.
|
|
152
|
+
If turso grows both, `_store.py` changes one import.
|
|
153
|
+
|
|
154
|
+
## Extraction: two passes, and why
|
|
155
|
+
|
|
156
|
+
- **`process()`**, the pack's intelligence layer — definitions with kinds, the
|
|
157
|
+
file's imports, docstrings and comments. What it will not give is
|
|
158
|
+
*references*: `SymbolInfo` says a function exists, never who calls it.
|
|
159
|
+
- **the tags query**, tree-sitter's own `tags.scm` per language — the data
|
|
160
|
+
GitHub's code navigation is built on. `@definition.*` and, crucially,
|
|
161
|
+
`@reference.call`.
|
|
162
|
+
|
|
163
|
+
Measured over `packages/ph-core/src/ph`: 136 files and 1.58 MiB through
|
|
164
|
+
`process()` in 219 ms, through the tags query in 108 ms. Indexing this
|
|
165
|
+
repository's core takes 2.0 s end to end including the SQLite writes, and
|
|
166
|
+
0.1 s when nothing has changed. Parsing twice is worth more than the parse it
|
|
167
|
+
saves.
|
|
168
|
+
|
|
169
|
+
Four details that are easy to get wrong, each handled once in `_extract` and
|
|
170
|
+
each pinned by a test:
|
|
171
|
+
|
|
172
|
+
- **`ProcessConfig` spans are 0-based** while tree-sitter points are 0-based and
|
|
173
|
+
every line pH shows a model is 1-based. Normalized on the way out, so nothing
|
|
174
|
+
downstream has to remember which pass a number came from. Getting this wrong
|
|
175
|
+
is an off-by-one pointer, which is what makes a model stop trusting a tool.
|
|
176
|
+
- **`SymbolInfo.doc` is always `None`.** The pack does not populate it —
|
|
177
|
+
measured against a file full of docstrings. Trusting it would have shipped an
|
|
178
|
+
index with an empty `doc` column and a `search` mode matching names only. The
|
|
179
|
+
prose is on two other channels: `result.docstrings`, which carries the pack's
|
|
180
|
+
own `associated_item`, and `result.comments` for doc comments (Rust `///`,
|
|
181
|
+
TypeScript block comments, Go `//`). Those have no association, so the rule is
|
|
182
|
+
positional and narrow — the comment's last line must be *immediately* above
|
|
183
|
+
the definition's first.
|
|
184
|
+
- **A comment node's `span.end_line` can be one past the line it occupies**,
|
|
185
|
+
because Rust's `///` node includes its trailing newline. Trusting it missed
|
|
186
|
+
the definition directly below *and* matched one a blank line away. The line
|
|
187
|
+
count comes from the comment's own text instead.
|
|
188
|
+
- **TypeScript's tags query is half the story.** Its grammar extends
|
|
189
|
+
javascript's and upstream splits the queries to match, so `class`, `function`
|
|
190
|
+
and `call` live in the *javascript* query — a `.ts` file matched against the
|
|
191
|
+
typescript query alone yields two tags and no calls. `INHERITS` fixes that.
|
|
192
|
+
|
|
193
|
+
## Per-language caveats
|
|
194
|
+
|
|
195
|
+
Both are pinned by tests, so they are known shapes rather than surprises:
|
|
196
|
+
|
|
197
|
+
- **C has definitions but no call references.** Its tags query ships no
|
|
198
|
+
`@reference.*` captures at all, so `callers`/`callees` are empty for C while
|
|
199
|
+
`search`, `define` and `entities` work.
|
|
200
|
+
- **A bare Ruby send is invisible.** `helper` without parentheses parses as an
|
|
201
|
+
identifier rather than a `call`, so only `helper()` becomes an edge.
|
|
202
|
+
|
|
203
|
+
Only `reference.call` is stored. The tags queries also emit `reference.type`,
|
|
204
|
+
`reference.class` and more; each is a real relationship, but folding them in
|
|
205
|
+
would make `callers` return every place that merely *mentions* a type.
|
|
206
|
+
|
|
207
|
+
## Where things live
|
|
208
|
+
|
|
209
|
+
- **the index**: `$PH_CACHE/code-graph/<digest of the workspace root>.db` — the
|
|
210
|
+
cache root because it is rebuildable from the source, and keyed by root so two
|
|
211
|
+
checkouts do not answer each other's questions. `path:` in the row's config
|
|
212
|
+
overrides it.
|
|
213
|
+
- **the grammars**: `$PH_CACHE/tree-sitter`, set by the row at mount.
|
|
214
|
+
|
|
215
|
+
That second one is not housekeeping. The pack materializes even its *bundled*
|
|
216
|
+
grammars into a writable cache on first use — the wheel ships them as an
|
|
217
|
+
archive, not as loadable libraries — and it **fails hard** when that directory
|
|
218
|
+
cannot be created, rather than falling back to the wheel. Its own default is
|
|
219
|
+
`$XDG_CACHE_HOME/…`, which is right on a laptop and wrong in a container with a
|
|
220
|
+
read-only `HOME`; both measured. So the row hands over the root pH already
|
|
221
|
+
designates for rebuildable artifacts, and refuses at mount with a `MountRefusal`
|
|
222
|
+
sentence — not a `pathlib` traceback — when even that is unwritable.
|
|
223
|
+
|
|
224
|
+
`TREE_SITTER_LANGUAGE_PACK_CACHE_DIR` still wins if an operator set it: their
|
|
225
|
+
spelling is what makes the variable mean anything.
|
|
226
|
+
|
|
227
|
+
One consequence worth expecting: the **first** index on a cold grammar cache
|
|
228
|
+
pays for materializing the grammar. Measured on `packages/ph-core/src/ph`, that
|
|
229
|
+
is 9 s cold against 2 s warm, and 0.1 s when nothing changed.
|
|
230
|
+
|
|
231
|
+
Provision it on purpose rather than during someone's turn:
|
|
232
|
+
|
|
233
|
+
```
|
|
234
|
+
/code-graph status # how many grammars are ready, and where they live
|
|
235
|
+
/code-graph install # load each one now, naming any that will not
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
A **command**, not a tool — a person asks the harness to provision, and it costs
|
|
239
|
+
no model turn. `phern doctor` reports the same path and the index's state without
|
|
240
|
+
mounting an agent. It matters less here than for `ph-text-index`, whose model is
|
|
241
|
+
a download rather than an unpack, but the question should have one answer per
|
|
242
|
+
plugin asked the same way.
|
|
243
|
+
|
|
244
|
+
## The skill
|
|
245
|
+
|
|
246
|
+
The package installs a `code-graph` skill, so an RLM under progressive
|
|
247
|
+
disclosure sees one line in its catalog and can read the page when it needs
|
|
248
|
+
it — index first, ask the question you actually have, then `read` the narrow
|
|
249
|
+
thing; and that a name-based graph reports ambiguity rather than resolving it.
|
|
250
|
+
|
|
251
|
+
Registered by the row rather than found by a directory scan, so it arrives
|
|
252
|
+
exactly when the tools do and leaves with them — `skills-progressive` ships an
|
|
253
|
+
empty `paths` on purpose, because a skill is something a distribution installs
|
|
254
|
+
deliberately (I7).
|
|
255
|
+
|
|
256
|
+
## Tests
|
|
257
|
+
|
|
258
|
+
`tests/test_code_graph.py` runs the real parser and the real SQLite index.
|
|
259
|
+
Three things it pins deliberately:
|
|
260
|
+
|
|
261
|
+
- **line spans against the file** — every span is checked by slicing the actual
|
|
262
|
+
source at it, not against an expected number, because a pointer that is off
|
|
263
|
+
by one still looks plausible in a diff;
|
|
264
|
+
- **the enclosing-symbol rule** — a call inside a nested closure belongs to the
|
|
265
|
+
closure, not the 40-line function around it;
|
|
266
|
+
- **incrementality** — a second index pass over an unchanged tree parses nothing
|
|
267
|
+
and a touched-but-unmodified file stays unchanged.
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# ph-code-graph
|
|
2
|
+
|
|
3
|
+
*`code_index` and `code_graph`: ask a codebase about its own shape, and get back
|
|
4
|
+
a `path:start-end` you can `read`.*
|
|
5
|
+
|
|
6
|
+
One row, `code-graph`, registering two tools. No Node, no Rust toolchain, no
|
|
7
|
+
submodule — tree-sitter through a Python wheel, the graph in stdlib `sqlite3`.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
phern --patch '{insert: [{id: code-graph, name: code-graph}]}' --profile llama \
|
|
11
|
+
-p "index packages/ph-core, then tell me what breaks if I change claim_slot"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## The two tools
|
|
15
|
+
|
|
16
|
+
**`code_index(paths, glob?, forget?)`** parses each file through `ctx.fs` and
|
|
17
|
+
records what it defines and references.
|
|
18
|
+
|
|
19
|
+
**Nothing watches the filesystem.** There is no daemon and no file watcher: the
|
|
20
|
+
index changes when `code_index` runs, and at no other moment. So after your own
|
|
21
|
+
edits, run it again — which is cheap by design, and cheap enough to be a habit.
|
|
22
|
+
|
|
23
|
+
**Incremental in two layers**, and the order matters:
|
|
24
|
+
|
|
25
|
+
1. *Does this file need opening?* Asked of **git or jj**, which already hold a
|
|
26
|
+
content-addressed tree — `ph.seams.changes`. A file the version control
|
|
27
|
+
vouches for is never read. Measured on `packages/ph-core/src/ph`: a re-index
|
|
28
|
+
of 137 unchanged files went from 0.10 s to **0.03 s**, with zero reads and
|
|
29
|
+
zero hashes, on a `tree_state` costing 7 ms for the whole 509-file repo.
|
|
30
|
+
2. *Is what I opened different?* Asked of the file's **sha256**, exactly as
|
|
31
|
+
before. Never an mtime — a checkout or a rebase moves timestamps without
|
|
32
|
+
moving content, and re-indexing because git touched them is a cost nobody
|
|
33
|
+
attributes correctly.
|
|
34
|
+
|
|
35
|
+
Layer 1 only decides what to open, so the guarantee layer 2 makes is untouched:
|
|
36
|
+
a skip is always justified by somebody's content hash, never by a clock. A tree
|
|
37
|
+
with no version control loses layer 1 and nothing else.
|
|
38
|
+
|
|
39
|
+
**`code_graph(mode, query?, …)`** answers six questions:
|
|
40
|
+
|
|
41
|
+
| `mode` | question | what comes back |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| `search` | "something about workspace tiers" | symbols ranked by FTS5 over name + docstring |
|
|
44
|
+
| `define` | "where exactly is `claim_slot`" | every definition of that name |
|
|
45
|
+
| `callers` | "what calls this" | the calling symbol **and the calling line** |
|
|
46
|
+
| `callees` | "what does this call" | the definitions it reaches |
|
|
47
|
+
| `impact` | "what breaks if I change this" | transitive callers, ring by ring |
|
|
48
|
+
| `entities` | "what is big here" | definitions by span, biggest first |
|
|
49
|
+
|
|
50
|
+
Every result carries `path:start-line-end-line`. That is the point of the
|
|
51
|
+
package: the model finds what matters in one call, then `read`s exactly that.
|
|
52
|
+
|
|
53
|
+
## Using it from the RLM
|
|
54
|
+
|
|
55
|
+
Nothing to do. Under Code Mode the model is handed one callable and a generated
|
|
56
|
+
SDK listing, and *every* registered tool is in that listing — so these arrive as
|
|
57
|
+
`await tools.code_index(...)` and `await tools.code_graph(...)` with no Code
|
|
58
|
+
Mode work in this package at all. The `rlm-indexed` profile is `rlm-stable` plus
|
|
59
|
+
this bundle and `ph-text-index`'s:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
phern --profile rlm-indexed --provider llama --model <model> --mode tui
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
This package registers a `ph.bundles` entry point, which is what lets `ph-app`
|
|
66
|
+
compose that profile without depending on this distribution — and what makes an
|
|
67
|
+
install missing it see no `rlm-indexed` rather than one that fails at mount.
|
|
68
|
+
|
|
69
|
+
## It is a name-based graph, and it says so
|
|
70
|
+
|
|
71
|
+
A reference records the *name* it used, and `callers`/`callees` join on that
|
|
72
|
+
name. Two `register` methods in two classes are one name to this index.
|
|
73
|
+
|
|
74
|
+
Every answer that could be ambiguous carries `definitions` — how many places
|
|
75
|
+
that name is defined — and the rendered text says so outright when it is above
|
|
76
|
+
one, so the model sees the ambiguity instead of being handed one of the
|
|
77
|
+
possibilities. `mode=define` is how it disambiguates.
|
|
78
|
+
|
|
79
|
+
Resolving properly means an import graph, scope, and per-language type
|
|
80
|
+
inference. In the Rust/Node CodeGraph that is 29,708 lines of TypeScript, and
|
|
81
|
+
this package deliberately did not port it. Name-based matching answers most of
|
|
82
|
+
what an agent actually asks and reports where it cannot — which is a better
|
|
83
|
+
trade than a resolved graph for one language.
|
|
84
|
+
|
|
85
|
+
## Why Python-native, and not a wrapper
|
|
86
|
+
|
|
87
|
+
The obvious plan was to submodule the Rust/Node CodeGraph and wrap its kernel
|
|
88
|
+
with maturin. The investigation said no, for three reasons in increasing order
|
|
89
|
+
of importance:
|
|
90
|
+
|
|
91
|
+
1. **`codegraph-kernel` is a `#[napi]` crate** — `crate-type = ["cdylib"]`
|
|
92
|
+
against `napi`/`napi-derive`. maturin builds PyO3/cffi extensions; wrapping
|
|
93
|
+
it means forking the crate to rewrite its boundary.
|
|
94
|
+
2. **It is only an extractor.** Its whole export surface is `extract_file`,
|
|
95
|
+
`contract_info`, `grammar_info` and two `cfnptr` helpers — "tree-sitter
|
|
96
|
+
parse+extract with one JS boundary crossing per file", in its own words. It
|
|
97
|
+
also has a complete TS/WASM fallback, so it is an accelerator, not the core.
|
|
98
|
+
3. **The intelligence is TypeScript**: `resolution/` 29,708 lines, `extraction/`
|
|
99
|
+
25,612, `mcp/` 13,940, `graph/` 6,023, `db/` 5,475 — 112,612 against the
|
|
100
|
+
kernel's 25,242. Wrapping the kernel gets a parser and leaves the product
|
|
101
|
+
behind.
|
|
102
|
+
|
|
103
|
+
And the parser is the part Python already has. `tree-sitter-language-pack` 1.16
|
|
104
|
+
ships **26 languages compiled into a 3.7 MB wheel, working offline** (python,
|
|
105
|
+
ts/tsx/js, rust, go, java, csharp, c, cpp, ruby, php, swift, kotlin, scala,
|
|
106
|
+
dart, lua, r, and more), with 371 available and the long tail fetched on
|
|
107
|
+
demand — which this row never triggers implicitly.
|
|
108
|
+
|
|
109
|
+
## Why `sqlite3` and not `pyturso`
|
|
110
|
+
|
|
111
|
+
pH's session log runs on turso, so this is the exception and the reason is
|
|
112
|
+
measured. Against a real 40 MB CodeGraph database (11,396 nodes, 36,830 edges),
|
|
113
|
+
`pyturso` served indexed lookups, joins and aggregates correctly, and then:
|
|
114
|
+
|
|
115
|
+
- **FTS5 is absent** — a `CREATE VIRTUAL TABLE … USING fts5` is invisible to
|
|
116
|
+
it, shadow tables included. That is `search`.
|
|
117
|
+
- **`Recursive CTEs are not yet supported`** — that is `impact`.
|
|
118
|
+
- **`json_extract` silently returns NULL** where SQLite returns the value:
|
|
119
|
+
`0.95` against `None`, same rows, same expression. A wrong answer with no
|
|
120
|
+
error is worse than a missing feature.
|
|
121
|
+
|
|
122
|
+
Two of the six modes here are exactly the two turso cannot serve. Stdlib
|
|
123
|
+
`sqlite3` (SQLite 3.45, FTS5 compiled in) costs no dependency and does all six.
|
|
124
|
+
If turso grows both, `_store.py` changes one import.
|
|
125
|
+
|
|
126
|
+
## Extraction: two passes, and why
|
|
127
|
+
|
|
128
|
+
- **`process()`**, the pack's intelligence layer — definitions with kinds, the
|
|
129
|
+
file's imports, docstrings and comments. What it will not give is
|
|
130
|
+
*references*: `SymbolInfo` says a function exists, never who calls it.
|
|
131
|
+
- **the tags query**, tree-sitter's own `tags.scm` per language — the data
|
|
132
|
+
GitHub's code navigation is built on. `@definition.*` and, crucially,
|
|
133
|
+
`@reference.call`.
|
|
134
|
+
|
|
135
|
+
Measured over `packages/ph-core/src/ph`: 136 files and 1.58 MiB through
|
|
136
|
+
`process()` in 219 ms, through the tags query in 108 ms. Indexing this
|
|
137
|
+
repository's core takes 2.0 s end to end including the SQLite writes, and
|
|
138
|
+
0.1 s when nothing has changed. Parsing twice is worth more than the parse it
|
|
139
|
+
saves.
|
|
140
|
+
|
|
141
|
+
Four details that are easy to get wrong, each handled once in `_extract` and
|
|
142
|
+
each pinned by a test:
|
|
143
|
+
|
|
144
|
+
- **`ProcessConfig` spans are 0-based** while tree-sitter points are 0-based and
|
|
145
|
+
every line pH shows a model is 1-based. Normalized on the way out, so nothing
|
|
146
|
+
downstream has to remember which pass a number came from. Getting this wrong
|
|
147
|
+
is an off-by-one pointer, which is what makes a model stop trusting a tool.
|
|
148
|
+
- **`SymbolInfo.doc` is always `None`.** The pack does not populate it —
|
|
149
|
+
measured against a file full of docstrings. Trusting it would have shipped an
|
|
150
|
+
index with an empty `doc` column and a `search` mode matching names only. The
|
|
151
|
+
prose is on two other channels: `result.docstrings`, which carries the pack's
|
|
152
|
+
own `associated_item`, and `result.comments` for doc comments (Rust `///`,
|
|
153
|
+
TypeScript block comments, Go `//`). Those have no association, so the rule is
|
|
154
|
+
positional and narrow — the comment's last line must be *immediately* above
|
|
155
|
+
the definition's first.
|
|
156
|
+
- **A comment node's `span.end_line` can be one past the line it occupies**,
|
|
157
|
+
because Rust's `///` node includes its trailing newline. Trusting it missed
|
|
158
|
+
the definition directly below *and* matched one a blank line away. The line
|
|
159
|
+
count comes from the comment's own text instead.
|
|
160
|
+
- **TypeScript's tags query is half the story.** Its grammar extends
|
|
161
|
+
javascript's and upstream splits the queries to match, so `class`, `function`
|
|
162
|
+
and `call` live in the *javascript* query — a `.ts` file matched against the
|
|
163
|
+
typescript query alone yields two tags and no calls. `INHERITS` fixes that.
|
|
164
|
+
|
|
165
|
+
## Per-language caveats
|
|
166
|
+
|
|
167
|
+
Both are pinned by tests, so they are known shapes rather than surprises:
|
|
168
|
+
|
|
169
|
+
- **C has definitions but no call references.** Its tags query ships no
|
|
170
|
+
`@reference.*` captures at all, so `callers`/`callees` are empty for C while
|
|
171
|
+
`search`, `define` and `entities` work.
|
|
172
|
+
- **A bare Ruby send is invisible.** `helper` without parentheses parses as an
|
|
173
|
+
identifier rather than a `call`, so only `helper()` becomes an edge.
|
|
174
|
+
|
|
175
|
+
Only `reference.call` is stored. The tags queries also emit `reference.type`,
|
|
176
|
+
`reference.class` and more; each is a real relationship, but folding them in
|
|
177
|
+
would make `callers` return every place that merely *mentions* a type.
|
|
178
|
+
|
|
179
|
+
## Where things live
|
|
180
|
+
|
|
181
|
+
- **the index**: `$PH_CACHE/code-graph/<digest of the workspace root>.db` — the
|
|
182
|
+
cache root because it is rebuildable from the source, and keyed by root so two
|
|
183
|
+
checkouts do not answer each other's questions. `path:` in the row's config
|
|
184
|
+
overrides it.
|
|
185
|
+
- **the grammars**: `$PH_CACHE/tree-sitter`, set by the row at mount.
|
|
186
|
+
|
|
187
|
+
That second one is not housekeeping. The pack materializes even its *bundled*
|
|
188
|
+
grammars into a writable cache on first use — the wheel ships them as an
|
|
189
|
+
archive, not as loadable libraries — and it **fails hard** when that directory
|
|
190
|
+
cannot be created, rather than falling back to the wheel. Its own default is
|
|
191
|
+
`$XDG_CACHE_HOME/…`, which is right on a laptop and wrong in a container with a
|
|
192
|
+
read-only `HOME`; both measured. So the row hands over the root pH already
|
|
193
|
+
designates for rebuildable artifacts, and refuses at mount with a `MountRefusal`
|
|
194
|
+
sentence — not a `pathlib` traceback — when even that is unwritable.
|
|
195
|
+
|
|
196
|
+
`TREE_SITTER_LANGUAGE_PACK_CACHE_DIR` still wins if an operator set it: their
|
|
197
|
+
spelling is what makes the variable mean anything.
|
|
198
|
+
|
|
199
|
+
One consequence worth expecting: the **first** index on a cold grammar cache
|
|
200
|
+
pays for materializing the grammar. Measured on `packages/ph-core/src/ph`, that
|
|
201
|
+
is 9 s cold against 2 s warm, and 0.1 s when nothing changed.
|
|
202
|
+
|
|
203
|
+
Provision it on purpose rather than during someone's turn:
|
|
204
|
+
|
|
205
|
+
```
|
|
206
|
+
/code-graph status # how many grammars are ready, and where they live
|
|
207
|
+
/code-graph install # load each one now, naming any that will not
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
A **command**, not a tool — a person asks the harness to provision, and it costs
|
|
211
|
+
no model turn. `phern doctor` reports the same path and the index's state without
|
|
212
|
+
mounting an agent. It matters less here than for `ph-text-index`, whose model is
|
|
213
|
+
a download rather than an unpack, but the question should have one answer per
|
|
214
|
+
plugin asked the same way.
|
|
215
|
+
|
|
216
|
+
## The skill
|
|
217
|
+
|
|
218
|
+
The package installs a `code-graph` skill, so an RLM under progressive
|
|
219
|
+
disclosure sees one line in its catalog and can read the page when it needs
|
|
220
|
+
it — index first, ask the question you actually have, then `read` the narrow
|
|
221
|
+
thing; and that a name-based graph reports ambiguity rather than resolving it.
|
|
222
|
+
|
|
223
|
+
Registered by the row rather than found by a directory scan, so it arrives
|
|
224
|
+
exactly when the tools do and leaves with them — `skills-progressive` ships an
|
|
225
|
+
empty `paths` on purpose, because a skill is something a distribution installs
|
|
226
|
+
deliberately (I7).
|
|
227
|
+
|
|
228
|
+
## Tests
|
|
229
|
+
|
|
230
|
+
`tests/test_code_graph.py` runs the real parser and the real SQLite index.
|
|
231
|
+
Three things it pins deliberately:
|
|
232
|
+
|
|
233
|
+
- **line spans against the file** — every span is checked by slicing the actual
|
|
234
|
+
source at it, not against an expected number, because a pointer that is off
|
|
235
|
+
by one still looks plausible in a diff;
|
|
236
|
+
- **the enclosing-symbol rule** — a call inside a nested closure belongs to the
|
|
237
|
+
closure, not the 40-line function around it;
|
|
238
|
+
- **incrementality** — a second index pass over an unchanged tree parses nothing
|
|
239
|
+
and a touched-but-unmodified file stays unchanged.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ph-code-graph"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "pH plugin: `code_index` and `code_graph`, a Python-native tree-sitter code graph with line-accurate answers."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Charles Tabor" }]
|
|
14
|
+
keywords = ["agent", "llm", "tree-sitter", "code-search", "code-graph"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Operating System :: POSIX :: Linux",
|
|
22
|
+
"Operating System :: MacOS",
|
|
23
|
+
"Topic :: Software Development",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
# Both are hard dependencies and both are small: `tree-sitter` is the binding
|
|
28
|
+
# and `tree-sitter-language-pack` is a 3.7 MB wheel with 26 languages compiled
|
|
29
|
+
# in (371 available, the rest fetched on demand — which this row never does
|
|
30
|
+
# implicitly; see `_extract.indexable`). No Rust toolchain, no Node, no
|
|
31
|
+
# submodule: the graph itself is stdlib `sqlite3`.
|
|
32
|
+
dependencies = ["ph-core==0.2.0", "tree-sitter>=0.25", "tree-sitter-language-pack>=1.16"]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/chastabor/pH"
|
|
36
|
+
Repository = "https://github.com/chastabor/pH"
|
|
37
|
+
Documentation = "https://github.com/chastabor/pH/blob/main/docs/README.md"
|
|
38
|
+
Issues = "https://github.com/chastabor/pH/issues"
|
|
39
|
+
|
|
40
|
+
[project.entry-points."ph.bundles"]
|
|
41
|
+
code-graph = "ph_code_graph:BUNDLE"
|
|
42
|
+
|
|
43
|
+
[project.entry-points."ph.plugins"]
|
|
44
|
+
code-graph = "ph_code_graph:apply"
|
|
45
|
+
|
|
46
|
+
[tool.uv.sources]
|
|
47
|
+
ph-core = { workspace = true }
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel]
|
|
50
|
+
packages = ["src/ph_code_graph"]
|