ph-code-graph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ph_code_graph/__init__.py +825 -0
- ph_code_graph/_extract.py +574 -0
- ph_code_graph/_store.py +638 -0
- ph_code_graph/bundle.yaml +21 -0
- ph_code_graph/py.typed +0 -0
- ph_code_graph/skills/code-graph/SKILL.md +66 -0
- ph_code_graph-0.1.0.dist-info/METADATA +267 -0
- ph_code_graph-0.1.0.dist-info/RECORD +11 -0
- ph_code_graph-0.1.0.dist-info/WHEEL +4 -0
- ph_code_graph-0.1.0.dist-info/entry_points.txt +5 -0
- ph_code_graph-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,825 @@
|
|
|
1
|
+
"""`code-graph` — a Python-native code graph, so the RLM can ask before it reads.
|
|
2
|
+
|
|
3
|
+
Two tools. `code_index` walks a tree through `ctx.fs`, parses it with
|
|
4
|
+
tree-sitter and records what each file defines and references. `code_graph`
|
|
5
|
+
answers five questions over that index — and answers every one of them with
|
|
6
|
+
`path:start-end`, so a hit is a pointer the agent can hand straight to `read`.
|
|
7
|
+
|
|
8
|
+
That last part is the whole point. An agent dropped into an unfamiliar
|
|
9
|
+
repository spends its first several thousand tokens reading files to discover
|
|
10
|
+
which two mattered. This turns that into one call.
|
|
11
|
+
|
|
12
|
+
## Python-native, and what that decided
|
|
13
|
+
|
|
14
|
+
The alternative was wrapping the Rust/Node CodeGraph, and the investigation said
|
|
15
|
+
no: that project's Rust kernel is a *tree-sitter extractor behind a Node-API
|
|
16
|
+
boundary* (its whole export surface is `extract_file`, `contract_info`,
|
|
17
|
+
`grammar_info` and two `cfnptr` helpers), while the intelligence — 29 708 lines
|
|
18
|
+
of cross-file resolution, plus graph, search and context layers — is TypeScript.
|
|
19
|
+
maturin could not build a `#[napi]` crate anyway. Wrapping it would have bought
|
|
20
|
+
a parser Python already has, and left the intelligence behind.
|
|
21
|
+
|
|
22
|
+
So: `tree-sitter-language-pack` for extraction — 26 languages bundled in a
|
|
23
|
+
3.7 MB wheel and working offline, 371 available — and `sqlite3` from the
|
|
24
|
+
standard library for the graph, because two of the five queries here are exactly
|
|
25
|
+
the two `pyturso` cannot serve (`_store` has the measurements).
|
|
26
|
+
|
|
27
|
+
## Name-based, and it says so
|
|
28
|
+
|
|
29
|
+
A reference records the *name* it used; `callers` and `callees` join on that
|
|
30
|
+
name. Two `register` methods in two classes are one name to this index. Every
|
|
31
|
+
result that could be ambiguous carries `definitions`, the number of places that
|
|
32
|
+
name is defined, so the model can see the ambiguity instead of being handed one
|
|
33
|
+
of them — and `code_graph mode=define` is how it disambiguates.
|
|
34
|
+
|
|
35
|
+
Resolving properly means an import graph, scope, and per-language type
|
|
36
|
+
inference. That is the 29 708 lines this package declined to port. Name-based
|
|
37
|
+
answers most of what an agent actually asks and reports where it cannot.
|
|
38
|
+
|
|
39
|
+
@module ph_code_graph
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import hashlib
|
|
45
|
+
import logging
|
|
46
|
+
from dataclasses import dataclass, field
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
from typing import Any, Literal
|
|
49
|
+
|
|
50
|
+
import anyio
|
|
51
|
+
from pydantic import Field
|
|
52
|
+
|
|
53
|
+
from ph.cordis import Context, MountRefusal, ServiceKey, plugin
|
|
54
|
+
from ph.json import JsonObject, as_int
|
|
55
|
+
from ph.keys import COMMANDS, FS, SKILLS, TOOLS
|
|
56
|
+
from ph.llm.types import ContentBlock
|
|
57
|
+
from ph.paths import default_cache_path, resolve_roots
|
|
58
|
+
from ph.seams._registry import contribute_item
|
|
59
|
+
from ph.seams.changes import tree_state
|
|
60
|
+
from ph.seams.commands import CommandContext, CommandDefinition
|
|
61
|
+
from ph.seams.diagnostics import Diagnostic, contribute
|
|
62
|
+
from ph.seams.skills import discover_skills
|
|
63
|
+
from ph.text import count_of
|
|
64
|
+
from ph.tools.definition import ToolModel, ToolOutput, ToolRunContext, define_tool, text_content
|
|
65
|
+
from ph.tools.errors import HarnessError
|
|
66
|
+
from ph.tools.presentation import simple_views
|
|
67
|
+
from ph.wire import WireModel
|
|
68
|
+
|
|
69
|
+
from ._extract import (
|
|
70
|
+
cache_release,
|
|
71
|
+
detect_language,
|
|
72
|
+
ensure,
|
|
73
|
+
extract,
|
|
74
|
+
indexable,
|
|
75
|
+
parseable,
|
|
76
|
+
readiness,
|
|
77
|
+
)
|
|
78
|
+
from ._store import CodeGraphStore, Hit, SymbolRow, digest_of
|
|
79
|
+
|
|
80
|
+
__all__ = ["BUNDLE", "CodeGraphSeam", "Config", "apply"]
|
|
81
|
+
|
|
82
|
+
BUNDLE = Path(__file__).parent / "bundle.yaml"
|
|
83
|
+
"""This distribution's profile layer, for the `ph.bundles` group.
|
|
84
|
+
|
|
85
|
+
Discovered rather than imported, which is what lets `ph-app` compose a
|
|
86
|
+
profile that layers this package without depending on it.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
log = logging.getLogger("ph_code_graph")
|
|
90
|
+
|
|
91
|
+
CODE_GRAPH_FAILED = "CODE_GRAPH_FAILED"
|
|
92
|
+
|
|
93
|
+
MISSING = (
|
|
94
|
+
"the code-graph row needs `tree-sitter` and `tree-sitter-language-pack`, which "
|
|
95
|
+
"ph-code-graph depends on; install them (`uv sync`) or remove the row"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
INDEX_DESCRIPTION = """Index a code tree so `code_graph` can answer questions about it.
|
|
99
|
+
|
|
100
|
+
Parses each file and records what it defines and what it references. Run it once
|
|
101
|
+
on a tree, then again after edits — it re-parses only files whose contents
|
|
102
|
+
changed, so re-running is cheap and keeps answers current.
|
|
103
|
+
|
|
104
|
+
Point it at a package rather than a whole monorepo. Non-code files are skipped."""
|
|
105
|
+
|
|
106
|
+
GRAPH_DESCRIPTION = """Ask about a codebase's structure instead of reading it.
|
|
107
|
+
|
|
108
|
+
Every answer carries `path:start-end`, so use this to find what matters and then
|
|
109
|
+
`read` exactly that.
|
|
110
|
+
|
|
111
|
+
- `search` — find symbols by name or docstring wording (fuzzy, ranked)
|
|
112
|
+
- `define` — where a name is defined, exactly
|
|
113
|
+
- `callers` — what calls this, with the calling line
|
|
114
|
+
- `callees` — what this calls
|
|
115
|
+
- `impact` — everything transitively affected by changing this, ring by ring
|
|
116
|
+
- `entities`— the largest definitions in a path, biggest first
|
|
117
|
+
|
|
118
|
+
Matching is by name, so a name defined in several places reports `definitions`
|
|
119
|
+
greater than 1 — use `define` to see which is which. Needs `code_index` to have
|
|
120
|
+
run first."""
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class Config(WireModel):
|
|
124
|
+
"""Row config for `code-graph`."""
|
|
125
|
+
|
|
126
|
+
path: str = ""
|
|
127
|
+
"""Where the index lives. Defaults to `$PH_CACHE/code-graph/<root digest>.db`.
|
|
128
|
+
|
|
129
|
+
Under the cache root because the index is **rebuildable** — the source is the
|
|
130
|
+
truth and this is derived, which is the lifecycle `$PH_CACHE` names (Q1).
|
|
131
|
+
Keyed by a digest of the workspace root so two checkouts do not share one
|
|
132
|
+
index and answer each other's questions."""
|
|
133
|
+
glob: str = "**/*"
|
|
134
|
+
"""What `code_index` takes from a directory when the call does not say.
|
|
135
|
+
|
|
136
|
+
Everything, because the language filter is the real one: a path whose
|
|
137
|
+
extension names no supported code language is skipped, so a `**/*` here
|
|
138
|
+
means "all the code" rather than "all the files"."""
|
|
139
|
+
languages: list[str] = Field(default_factory=list)
|
|
140
|
+
"""Restrict indexing to these languages. Empty means every bundled one.
|
|
141
|
+
|
|
142
|
+
A monorepo with a `node_modules` of vendored JavaScript and one Python
|
|
143
|
+
package wants `[python]`, and saying so is cheaper than a glob that has to
|
|
144
|
+
describe the same intent negatively."""
|
|
145
|
+
max_bytes: int = 2 * 1024 * 1024
|
|
146
|
+
"""The largest file parsed. One past it is skipped and reported, not an
|
|
147
|
+
error: a tree walk should not fail because it found a minified bundle or a
|
|
148
|
+
generated parser, and a caller who never learns what was skipped cannot tell
|
|
149
|
+
a thin graph from a broken one."""
|
|
150
|
+
max_files: int = 20_000
|
|
151
|
+
"""How many files one `code_index` call will consider."""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --------------------------------------------------------------------- seam ----
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
CODE_GRAPH: ServiceKey[CodeGraphSeam] = ServiceKey("code_graph")
|
|
158
|
+
"""The code graph, for `/code-graph` and the tools it backs."""
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@dataclass(slots=True)
|
|
162
|
+
class CodeGraphSeam:
|
|
163
|
+
"""The service published as `ctx.code_graph`.
|
|
164
|
+
|
|
165
|
+
Holds the store and the lock. The lock is why `code_index` declares itself
|
|
166
|
+
not concurrency-safe: two calls indexing overlapping trees would interleave
|
|
167
|
+
their per-file transactions, and while SQLite would keep each one atomic the
|
|
168
|
+
*pair* would report counts neither of them produced.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
ctx: Context
|
|
172
|
+
config: Config
|
|
173
|
+
grammars: Path
|
|
174
|
+
"""Where the tree-sitter grammar cache was pointed. See `_extract.use_cache`."""
|
|
175
|
+
_lock: anyio.Lock = field(default_factory=anyio.Lock)
|
|
176
|
+
|
|
177
|
+
def store_for(self, root: Path) -> CodeGraphStore:
|
|
178
|
+
"""This workspace's index. One file per root, and **nothing prunes them.**
|
|
179
|
+
|
|
180
|
+
Keyed per root on purpose — a worktree at another revision holds
|
|
181
|
+
different line numbers, so sharing one index would hand out pointers
|
|
182
|
+
that are quietly stale. The cost is that under a profile whose children
|
|
183
|
+
run in worktrees, every throwaway tree leaves a database behind, and
|
|
184
|
+
`phern doctor` names only the current one. Stated here rather than left to
|
|
185
|
+
be discovered, which is this codebase's rule for a cache nothing
|
|
186
|
+
collects (`ph.seams.uploads` says the same about attachments): deleting
|
|
187
|
+
`$PH_CACHE/code-graph` reclaims all of them and costs a re-index.
|
|
188
|
+
"""
|
|
189
|
+
digest = hashlib.sha256(str(root).encode("utf-8")).hexdigest()[:16]
|
|
190
|
+
return CodeGraphStore(
|
|
191
|
+
path=default_cache_path(self.config.path, "code-graph", f"{digest}.db")
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
def locked(self) -> anyio.Lock:
|
|
195
|
+
return self._lock
|
|
196
|
+
|
|
197
|
+
def report(self) -> list[tuple[str, str]]:
|
|
198
|
+
"""`phern doctor`'s section."""
|
|
199
|
+
import tree_sitter
|
|
200
|
+
|
|
201
|
+
# No `hasattr` guard: the row declares `inject=[FS]`, so `ctx.fs` is
|
|
202
|
+
# present for as long as it is active — and a `Path.cwd()` fallback
|
|
203
|
+
# answered with an index keyed to the process directory rather than the
|
|
204
|
+
# workspace, which is worse than the traceback it avoided.
|
|
205
|
+
root = self.ctx.require(FS).root_for(None)
|
|
206
|
+
store = self.store_for(root)
|
|
207
|
+
rows = [
|
|
208
|
+
("tree-sitter", getattr(tree_sitter, "__version__", "installed")),
|
|
209
|
+
("grammars", str(self.grammars)),
|
|
210
|
+
("index", str(store.path)),
|
|
211
|
+
]
|
|
212
|
+
if not store.exists():
|
|
213
|
+
rows.append(("state", "not built — run code_index"))
|
|
214
|
+
return rows
|
|
215
|
+
stats = store.stats()
|
|
216
|
+
rows.extend(
|
|
217
|
+
[
|
|
218
|
+
("files", str(stats["files"])),
|
|
219
|
+
("symbols", str(stats["symbols"])),
|
|
220
|
+
("references", str(stats["refs"])),
|
|
221
|
+
("languages", ", ".join(stats["languages"][:8]) or "none"),
|
|
222
|
+
]
|
|
223
|
+
)
|
|
224
|
+
return rows
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# ------------------------------------------------------------------ schemas ----
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class IndexArgs(ToolModel):
|
|
231
|
+
paths: list[str] = Field(
|
|
232
|
+
default_factory=lambda: ["."],
|
|
233
|
+
description="Directories or files to index. Relative to the workspace root.",
|
|
234
|
+
)
|
|
235
|
+
glob: str | None = Field(
|
|
236
|
+
None, description="Which files to take from a directory. Defaults to the row's setting."
|
|
237
|
+
)
|
|
238
|
+
forget: bool = Field(False, description="Remove these paths from the index instead.")
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class SkippedValue(ToolModel):
|
|
242
|
+
path: str
|
|
243
|
+
reason: str
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class IndexValue(ToolModel):
|
|
247
|
+
indexed: int
|
|
248
|
+
"""Files parsed this call — changed or new only."""
|
|
249
|
+
unchanged: int
|
|
250
|
+
"""Files already current, by content digest."""
|
|
251
|
+
removed: int
|
|
252
|
+
symbols: int
|
|
253
|
+
"""Symbols written this call."""
|
|
254
|
+
skipped: list[SkippedValue]
|
|
255
|
+
total_files: int
|
|
256
|
+
total_symbols: int
|
|
257
|
+
total_refs: int
|
|
258
|
+
languages: list[str]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
class GraphArgs(ToolModel):
|
|
262
|
+
mode: Literal["search", "define", "callers", "callees", "impact", "entities"] = Field(
|
|
263
|
+
"search", description="Which question to ask. See the tool description."
|
|
264
|
+
)
|
|
265
|
+
query: str | None = Field(
|
|
266
|
+
None,
|
|
267
|
+
description=(
|
|
268
|
+
"The symbol name for define/callers/callees/impact, or the search "
|
|
269
|
+
"wording for search. Not used by entities."
|
|
270
|
+
),
|
|
271
|
+
)
|
|
272
|
+
path: str | None = Field(None, description="For entities: restrict to this file or directory.")
|
|
273
|
+
kind: str = Field(
|
|
274
|
+
"any", description="For entities: keep only this kind (function, class, method, ...)."
|
|
275
|
+
)
|
|
276
|
+
distance: int = Field(2, ge=1, le=6, description="For impact: how many hops to follow.")
|
|
277
|
+
limit: int = Field(30, ge=1, le=300, description="Return at most this many results.")
|
|
278
|
+
offset: int = Field(0, ge=0, description="For entities: skip this many (paging).")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class SymbolValue(ToolModel):
|
|
282
|
+
name: str
|
|
283
|
+
kind: str
|
|
284
|
+
path: str
|
|
285
|
+
start_line: int
|
|
286
|
+
end_line: int
|
|
287
|
+
lines: int
|
|
288
|
+
doc: str | None = None
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class EdgeValue(SymbolValue):
|
|
292
|
+
ref_line: int
|
|
293
|
+
"""The line the reference is written on — what to open."""
|
|
294
|
+
ref_path: str
|
|
295
|
+
"""Which file `ref_line` is in. **Not always `path`**: for `callees` the
|
|
296
|
+
symbol is the callee's definition while the reference is a line in the
|
|
297
|
+
caller. See `_store.Hit`."""
|
|
298
|
+
via: str
|
|
299
|
+
"""The name the edge was written as."""
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class RingValue(ToolModel):
|
|
303
|
+
distance: int
|
|
304
|
+
symbols: list[SymbolValue]
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
class GraphValue(ToolModel):
|
|
308
|
+
mode: str
|
|
309
|
+
query: str | None
|
|
310
|
+
definitions: int
|
|
311
|
+
"""How many places `query` is defined. **Above 1 means the answer is
|
|
312
|
+
ambiguous** — this index matches by name; see the module docstring."""
|
|
313
|
+
total: int
|
|
314
|
+
offset: int
|
|
315
|
+
truncated: bool
|
|
316
|
+
indexed_files: int
|
|
317
|
+
symbols: list[SymbolValue] = Field(default_factory=list)
|
|
318
|
+
edges: list[EdgeValue] = Field(default_factory=list)
|
|
319
|
+
rings: list[RingValue] = Field(default_factory=list)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# ------------------------------------------------------------------- render ----
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _where(one: dict[str, Any]) -> str:
|
|
326
|
+
return f"{one['path']}:{one['start_line']}-{one['end_line']}"
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _doc(one: dict[str, Any]) -> str:
|
|
330
|
+
doc = (one.get("doc") or "").strip().splitlines()
|
|
331
|
+
return f" — {doc[0][:70]}" if doc else ""
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _render_index(args: JsonObject, value: Any) -> list[ContentBlock]: # noqa: ANN401
|
|
335
|
+
if args.get("forget"):
|
|
336
|
+
return text_content(
|
|
337
|
+
f"Removed {count_of(value['removed'], 'file')} from the index. "
|
|
338
|
+
f"It now holds {count_of(value['total_symbols'], 'symbol')} "
|
|
339
|
+
f"from {count_of(value['total_files'], 'file')}."
|
|
340
|
+
)
|
|
341
|
+
lines = [
|
|
342
|
+
f"Indexed {count_of(value['indexed'], 'file')} "
|
|
343
|
+
f"({value['unchanged']} already current), {count_of(value['symbols'], 'symbol')}."
|
|
344
|
+
]
|
|
345
|
+
for skipped in value["skipped"][:20]:
|
|
346
|
+
lines.append(f" skipped {skipped['path']}: {skipped['reason']}")
|
|
347
|
+
if len(value["skipped"]) > 20:
|
|
348
|
+
lines.append(f" [and {len(value['skipped']) - 20} more skipped]")
|
|
349
|
+
lines.append(
|
|
350
|
+
f"The graph holds {count_of(value['total_symbols'], 'symbol')} and "
|
|
351
|
+
f"{count_of(value['total_refs'], 'reference')} across "
|
|
352
|
+
f"{count_of(value['total_files'], 'file')} "
|
|
353
|
+
f"({', '.join(value['languages'][:6]) or 'no languages'})."
|
|
354
|
+
)
|
|
355
|
+
return text_content("\n".join(lines))
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _ambiguity(value: Any) -> str: # noqa: ANN401
|
|
359
|
+
if value["definitions"] <= 1:
|
|
360
|
+
return ""
|
|
361
|
+
return (
|
|
362
|
+
f"\n[{value['query']!r} is defined in {value['definitions']} places and this "
|
|
363
|
+
"index matches by name, so these edges may belong to more than one of them; "
|
|
364
|
+
"`mode=define` lists them]"
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _render_graph(args: JsonObject, value: Any) -> list[ContentBlock]: # noqa: ANN401
|
|
369
|
+
mode = value["mode"]
|
|
370
|
+
if mode in ("search", "define"):
|
|
371
|
+
if not value["symbols"]:
|
|
372
|
+
return text_content(
|
|
373
|
+
f"Nothing matched {value['query']!r} in "
|
|
374
|
+
f"{count_of(value['indexed_files'], 'indexed file')}. "
|
|
375
|
+
"Run `code_index` first if this tree was never indexed."
|
|
376
|
+
)
|
|
377
|
+
lines = [f"{count_of(value['total'], 'match', 'matches')} for {value['query']!r}:"]
|
|
378
|
+
for one in value["symbols"]:
|
|
379
|
+
lines.append(f" {one['kind']:<10} {one['name']:<24} {_where(one)}{_doc(one)}")
|
|
380
|
+
elif mode in ("callers", "callees"):
|
|
381
|
+
if not value["edges"]:
|
|
382
|
+
verb = "calls" if mode == "callers" else "is called by"
|
|
383
|
+
return text_content(
|
|
384
|
+
f"Nothing {verb} {value['query']!r} in the index."
|
|
385
|
+
+ (
|
|
386
|
+
""
|
|
387
|
+
if value["definitions"]
|
|
388
|
+
else f" No symbol named {value['query']!r} is indexed at all."
|
|
389
|
+
)
|
|
390
|
+
)
|
|
391
|
+
header = (
|
|
392
|
+
f"calling {value['query']!r}"
|
|
393
|
+
if mode == "callers"
|
|
394
|
+
else f"where {value['query']!r} calls out"
|
|
395
|
+
)
|
|
396
|
+
lines = [f"{count_of(value['total'], 'site')} — {header}:"]
|
|
397
|
+
for one in value["edges"]:
|
|
398
|
+
# `ref_path`, not `path`: the reference and the definition are in
|
|
399
|
+
# different files for `callees`, and printing the line against the
|
|
400
|
+
# wrong one is a pointer to nothing (`_store.Hit`).
|
|
401
|
+
at = f"{one['ref_path']}:{one['ref_line']}"
|
|
402
|
+
lines.append(
|
|
403
|
+
f" {one['name']:<24} {at:<44} (defined {_where(one)})"
|
|
404
|
+
+ (f" via {one['via']}" if one["via"] != value["query"] else "")
|
|
405
|
+
)
|
|
406
|
+
elif mode == "impact":
|
|
407
|
+
if not value["rings"]:
|
|
408
|
+
return text_content(
|
|
409
|
+
f"Nothing depends on {value['query']!r} within "
|
|
410
|
+
f"{count_of(as_int(args.get('distance'), 2), 'hop')}."
|
|
411
|
+
)
|
|
412
|
+
lines = [f"Changing {value['query']!r} reaches:"]
|
|
413
|
+
for ring in value["rings"]:
|
|
414
|
+
lines.append(
|
|
415
|
+
f"\n {count_of(ring['distance'], 'hop')} — "
|
|
416
|
+
f"{count_of(len(ring['symbols']), 'symbol')}:"
|
|
417
|
+
)
|
|
418
|
+
for one in ring["symbols"]:
|
|
419
|
+
lines.append(f" {one['name']:<24} {_where(one)}")
|
|
420
|
+
else:
|
|
421
|
+
if not value["symbols"]:
|
|
422
|
+
return text_content("No definitions matched.")
|
|
423
|
+
lines = [f"{count_of(value['total'], 'definition')}, biggest first:"]
|
|
424
|
+
lines.append("\nlines kind name where")
|
|
425
|
+
for one in value["symbols"]:
|
|
426
|
+
lines.append(f"{one['lines']:>5} {one['kind']:<10} {one['name']:<24} {_where(one)}")
|
|
427
|
+
if value["truncated"]:
|
|
428
|
+
shown = value["offset"] + max(len(value["symbols"]), len(value["edges"]))
|
|
429
|
+
lines.append(f"\n[{value['total'] - shown} more; re-run with offset={shown}]")
|
|
430
|
+
text = "\n".join(lines) + _ambiguity(value)
|
|
431
|
+
return text_content(text)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
# --------------------------------------------------------------------- body ----
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
DEFAULT_LANGUAGES = (
|
|
438
|
+
"python",
|
|
439
|
+
"typescript",
|
|
440
|
+
"tsx",
|
|
441
|
+
"javascript",
|
|
442
|
+
"rust",
|
|
443
|
+
"go",
|
|
444
|
+
"java",
|
|
445
|
+
"csharp",
|
|
446
|
+
"c",
|
|
447
|
+
"cpp",
|
|
448
|
+
"ruby",
|
|
449
|
+
"php",
|
|
450
|
+
"swift",
|
|
451
|
+
"kotlin",
|
|
452
|
+
"scala",
|
|
453
|
+
"dart",
|
|
454
|
+
"lua",
|
|
455
|
+
"r",
|
|
456
|
+
"sql",
|
|
457
|
+
"bash",
|
|
458
|
+
)
|
|
459
|
+
"""What `/code-graph install|status` covers when the row names no languages.
|
|
460
|
+
|
|
461
|
+
A convenience list, and **filtered through `indexable` before use** — `bash` and
|
|
462
|
+
`sql` are in it and have no tags query, so an unfiltered list reported them
|
|
463
|
+
ready and then handed the extractor a language it could not use. Not read by the
|
|
464
|
+
indexer, which asks `indexable`/`local` per file, so a language outside this
|
|
465
|
+
list still works when the pack has it.
|
|
466
|
+
"""
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def offer_skills(ctx: Context) -> None:
|
|
470
|
+
"""Install this package's `SKILL.md` files, if a skills registry is mounted.
|
|
471
|
+
|
|
472
|
+
`discover_skills` already globs `<root>/*/SKILL.md`, validates each one and
|
|
473
|
+
skips a malformed one with a logged reason — the whole of what a
|
|
474
|
+
hand-written loader was doing here in 28 lines, duplicated byte-for-byte
|
|
475
|
+
into the sibling package. Reaching for the seam's own reader instead is the
|
|
476
|
+
rule `contribute_via`'s docstring states about itself.
|
|
477
|
+
|
|
478
|
+
**Through `contribute_via`, so the skill arrives exactly when the plugin
|
|
479
|
+
does** and leaves with it. `skills-progressive` ships an empty `paths` on
|
|
480
|
+
purpose — scanning a well-known directory would make "install a skill" mean
|
|
481
|
+
"drop a file somewhere", and a skill is something a distribution installs
|
|
482
|
+
deliberately (I7). A distribution registering its own is that act.
|
|
483
|
+
|
|
484
|
+
Only the one-line description rides the prompt every turn; the body stays on
|
|
485
|
+
disk until the model asks for it by name (G9).
|
|
486
|
+
"""
|
|
487
|
+
root = Path(__file__).parent / "skills"
|
|
488
|
+
for skill in discover_skills([str(root)], source="ph-code-graph"):
|
|
489
|
+
contribute_item(ctx, SKILLS, skill, label=f"skill({skill.name})")
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
@plugin("code-graph", inject=[TOOLS, FS], config=Config)
|
|
493
|
+
async def apply(ctx: Context, config: Config) -> None:
|
|
494
|
+
"""Mount the seam and register both tools."""
|
|
495
|
+
import importlib.util
|
|
496
|
+
|
|
497
|
+
for module in ("tree_sitter", "tree_sitter_language_pack"):
|
|
498
|
+
if importlib.util.find_spec(module) is None:
|
|
499
|
+
raise MountRefusal(MISSING)
|
|
500
|
+
|
|
501
|
+
# Before anything parses. The pack materialises even its *bundled* grammars
|
|
502
|
+
# into a writable cache directory and fails hard without one, so the row
|
|
503
|
+
# names a path it can vouch for rather than inheriting whatever `HOME`
|
|
504
|
+
# happens to be — see `use_cache`.
|
|
505
|
+
grammar_cache = resolve_roots().cache / "tree-sitter"
|
|
506
|
+
try:
|
|
507
|
+
# Through `ctx.effect` and in a worker thread, for two reasons that
|
|
508
|
+
# happen to share one line. The effect is so the library's
|
|
509
|
+
# process-global cache setting unwinds with this row rather than
|
|
510
|
+
# outliving it (§4.9, I2 — see `cache_release`); the thread is because
|
|
511
|
+
# `use_cache` imports `tree_sitter_language_pack`, a 20 ms dlopen that
|
|
512
|
+
# a profile mounting many rows would otherwise serialise on the loop.
|
|
513
|
+
await ctx.effect(
|
|
514
|
+
lambda: anyio.to_thread.run_sync(cache_release, grammar_cache),
|
|
515
|
+
label="tree-sitter-cache",
|
|
516
|
+
)
|
|
517
|
+
grammars = grammar_cache
|
|
518
|
+
except OSError as error:
|
|
519
|
+
# `MountRefusal`, not the `OSError`: every command that mounts a profile
|
|
520
|
+
# turns this one type into a sentence and an exit code, and leaves
|
|
521
|
+
# anything else as the traceback a bug deserves. A read-only `$PH_CACHE`
|
|
522
|
+
# is a deployment fact an operator can fix, not a bug — measured inside
|
|
523
|
+
# a sandbox that made `~/.cache` read-only, where this arrived as
|
|
524
|
+
# fourteen frames of `pathlib`.
|
|
525
|
+
raise MountRefusal(
|
|
526
|
+
f"code-graph cannot write the tree-sitter grammar cache at {error.filename}: "
|
|
527
|
+
f"{error.strerror}. The grammars are materialised there on first use, so this "
|
|
528
|
+
"path must be writable — point $PH_CACHE somewhere it is, or set "
|
|
529
|
+
"TREE_SITTER_LANGUAGE_PACK_CACHE_DIR."
|
|
530
|
+
) from error
|
|
531
|
+
|
|
532
|
+
seam = CodeGraphSeam(ctx=ctx, config=config, grammars=grammars)
|
|
533
|
+
ctx.provide(CODE_GRAPH, seam)
|
|
534
|
+
|
|
535
|
+
def store(run: ToolRunContext) -> CodeGraphStore:
|
|
536
|
+
return seam.store_for(ctx.require(FS).root_for(run.agent))
|
|
537
|
+
|
|
538
|
+
async def index_tool(args: IndexArgs, run: ToolRunContext) -> dict[str, Any]:
|
|
539
|
+
fs = ctx.require(FS)
|
|
540
|
+
book = store(run)
|
|
541
|
+
paths = await fs.collect(
|
|
542
|
+
args.paths,
|
|
543
|
+
args.glob or config.glob,
|
|
544
|
+
scope=run.scope,
|
|
545
|
+
limit=config.max_files,
|
|
546
|
+
agent=run.agent,
|
|
547
|
+
)
|
|
548
|
+
skipped: list[dict[str, str]] = []
|
|
549
|
+
indexed = unchanged = symbols = removed = 0
|
|
550
|
+
|
|
551
|
+
async with seam.locked():
|
|
552
|
+
await anyio.to_thread.run_sync(book.prepare)
|
|
553
|
+
if args.forget:
|
|
554
|
+
# Falls through to the one `stats` read and the one return
|
|
555
|
+
# below: `indexed`, `unchanged` and `symbols` are already 0 and
|
|
556
|
+
# `skipped` already empty, so a second copy of the payload was
|
|
557
|
+
# eleven lines saying that again — and a second shape for one
|
|
558
|
+
# `IndexValue` schema, in the mode with no test asserting it.
|
|
559
|
+
removed = await anyio.to_thread.run_sync(book.forget, paths)
|
|
560
|
+
paths = []
|
|
561
|
+
known = await anyio.to_thread.run_sync(book.known)
|
|
562
|
+
# **Ask the version control what changed before reading anything.**
|
|
563
|
+
# `tree_state` never raises and answers an empty state for a tree
|
|
564
|
+
# with no backend, so the loop below is correct either way — it just
|
|
565
|
+
# reads every file when nothing can vouch for one. Which backend
|
|
566
|
+
# answers is the workspace provider's to say (`ph.seams.changes`).
|
|
567
|
+
parsers: dict[str, bool] = {}
|
|
568
|
+
stored_token = await anyio.to_thread.run_sync(book.token)
|
|
569
|
+
state = await tree_state(ctx, fs.root_for(run.agent), since=stored_token)
|
|
570
|
+
for path in paths:
|
|
571
|
+
run.raise_if_cancelled()
|
|
572
|
+
language = detect_language(path)
|
|
573
|
+
# Derived from the pack rather than checked against a name
|
|
574
|
+
# list: `.txt`, `.ini`, `.proto` and a dozen others are
|
|
575
|
+
# languages the detector claims and the tags queries do not
|
|
576
|
+
# cover, and the row's default `glob` reaches all of them.
|
|
577
|
+
if language is None or not indexable(language):
|
|
578
|
+
continue
|
|
579
|
+
if config.languages and language not in config.languages:
|
|
580
|
+
continue
|
|
581
|
+
if language not in parsers:
|
|
582
|
+
# **One hop per language, not per file.** The first file of a
|
|
583
|
+
# language is worth a worker thread — `parseable`
|
|
584
|
+
# materialises a grammar out of the wheel, 6.4 s the first
|
|
585
|
+
# time — and every later file of it is a cached boolean, so
|
|
586
|
+
# the 60.6 µs hop was the entire cost: 1.2 s across a
|
|
587
|
+
# 20 000-file tree, which is the whole saving the change
|
|
588
|
+
# filter exists to deliver, handed back.
|
|
589
|
+
#
|
|
590
|
+
# Keyed per call rather than `@cache`d for the process: a
|
|
591
|
+
# person who runs `/code-graph install` between two calls has
|
|
592
|
+
# to see the grammar that appeared.
|
|
593
|
+
parsers[language] = await anyio.to_thread.run_sync(parseable, language)
|
|
594
|
+
if not parsers[language]:
|
|
595
|
+
skipped.append(
|
|
596
|
+
{
|
|
597
|
+
"path": path,
|
|
598
|
+
"reason": (
|
|
599
|
+
f"no {language} parser available — a language outside the "
|
|
600
|
+
"wheel's bundled set comes from GitHub; /code-graph install"
|
|
601
|
+
),
|
|
602
|
+
}
|
|
603
|
+
)
|
|
604
|
+
continue
|
|
605
|
+
stored_digest, stored_vcs = known.get(path, ("", ""))
|
|
606
|
+
if stored_digest and state.vouches_for(path, stored_vcs):
|
|
607
|
+
# Proved unchanged by git or jj, so the file is never opened.
|
|
608
|
+
# The saving is the whole read: 5.2 ms of I/O plus 3.0 ms of
|
|
609
|
+
# sha256 per 136 files, which is ~1.2 s on a 20 000-file tree.
|
|
610
|
+
unchanged += 1
|
|
611
|
+
continue
|
|
612
|
+
refused = fs.skip_reason(path, max_bytes=config.max_bytes, agent=run.agent)
|
|
613
|
+
if refused:
|
|
614
|
+
skipped.append({"path": path, "reason": refused})
|
|
615
|
+
continue
|
|
616
|
+
slice_ = await fs.read(
|
|
617
|
+
path,
|
|
618
|
+
limit=None,
|
|
619
|
+
scope=run.scope,
|
|
620
|
+
agent=run.agent,
|
|
621
|
+
session=run.session,
|
|
622
|
+
)
|
|
623
|
+
# The content hash stays the authority on *whether* a file
|
|
624
|
+
# changed — the filter above only decides whether to open it, so
|
|
625
|
+
# the "content, not clock" guarantee is untouched.
|
|
626
|
+
digest = digest_of(slice_.text)
|
|
627
|
+
if stored_digest == digest:
|
|
628
|
+
unchanged += 1
|
|
629
|
+
continue
|
|
630
|
+
try:
|
|
631
|
+
extraction = await anyio.to_thread.run_sync(
|
|
632
|
+
extract, path, slice_.text, language
|
|
633
|
+
)
|
|
634
|
+
except Exception as error:
|
|
635
|
+
# Reported and skipped rather than raised: a tree walk that
|
|
636
|
+
# died on one unparseable file would be the failure mode the
|
|
637
|
+
# package this replaced actually had.
|
|
638
|
+
log.debug("ph_code_graph: %s did not parse", path, exc_info=True)
|
|
639
|
+
skipped.append({"path": path, "reason": f"did not parse ({error})"})
|
|
640
|
+
continue
|
|
641
|
+
symbols += await anyio.to_thread.run_sync(
|
|
642
|
+
book.put, path, digest, extraction, state.id_for(path)
|
|
643
|
+
)
|
|
644
|
+
indexed += 1
|
|
645
|
+
# Stored last, and only after the loop: a token recorded before the
|
|
646
|
+
# writes would, on a crash between the two, vouch for files this run
|
|
647
|
+
# never actually indexed.
|
|
648
|
+
if state.token:
|
|
649
|
+
await anyio.to_thread.run_sync(book.remember, state.token)
|
|
650
|
+
stats = await anyio.to_thread.run_sync(book.stats)
|
|
651
|
+
|
|
652
|
+
return {
|
|
653
|
+
"indexed": indexed,
|
|
654
|
+
"unchanged": unchanged,
|
|
655
|
+
"removed": removed,
|
|
656
|
+
"symbols": symbols,
|
|
657
|
+
"skipped": skipped,
|
|
658
|
+
"total_files": stats["files"],
|
|
659
|
+
"total_symbols": stats["symbols"],
|
|
660
|
+
"total_refs": stats["refs"],
|
|
661
|
+
"languages": stats["languages"],
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
async def graph_tool(args: GraphArgs, run: ToolRunContext) -> dict[str, Any]:
|
|
665
|
+
book = store(run)
|
|
666
|
+
if not await anyio.to_thread.run_sync(book.exists):
|
|
667
|
+
raise HarnessError(
|
|
668
|
+
"no code index exists for this workspace yet; run `code_index` first",
|
|
669
|
+
CODE_GRAPH_FAILED,
|
|
670
|
+
)
|
|
671
|
+
if args.mode != "entities" and not args.query:
|
|
672
|
+
raise HarnessError(f"mode={args.mode} needs `query`", CODE_GRAPH_FAILED)
|
|
673
|
+
|
|
674
|
+
# `file_count`, not `stats()`: the header needs one number and `stats`
|
|
675
|
+
# counts symbols and refs and groups files by language — three full
|
|
676
|
+
# scans per query, growing with the corpus (1.97 ms against 0.22 ms on a
|
|
677
|
+
# 2 000-file index).
|
|
678
|
+
indexed = await anyio.to_thread.run_sync(book.file_count)
|
|
679
|
+
name = args.query or ""
|
|
680
|
+
# `COUNT(*)`, not `len(define(name, 100))`, which materialised a hundred
|
|
681
|
+
# rows and then reported the *cap* as the count for anything past it.
|
|
682
|
+
defined = (
|
|
683
|
+
await anyio.to_thread.run_sync(book.definition_count, name)
|
|
684
|
+
if args.mode != "search" and name
|
|
685
|
+
else 0
|
|
686
|
+
)
|
|
687
|
+
body: dict[str, Any] = {
|
|
688
|
+
"symbols": [],
|
|
689
|
+
"edges": [],
|
|
690
|
+
"rings": [],
|
|
691
|
+
"total": 0,
|
|
692
|
+
"truncated": False,
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
if args.mode == "search":
|
|
696
|
+
found = await anyio.to_thread.run_sync(book.search, _fts(name), args.limit)
|
|
697
|
+
body |= {"symbols": _symbols(found), "total": len(found)}
|
|
698
|
+
elif args.mode == "define":
|
|
699
|
+
found = await anyio.to_thread.run_sync(book.define, name, args.limit)
|
|
700
|
+
body |= {"symbols": _symbols(found), "total": len(found)}
|
|
701
|
+
elif args.mode in ("callers", "callees"):
|
|
702
|
+
call = book.callers if args.mode == "callers" else book.callees
|
|
703
|
+
hits: list[Hit] = await anyio.to_thread.run_sync(call, name, args.limit + 1)
|
|
704
|
+
body |= {
|
|
705
|
+
"edges": [one.as_value() for one in hits[: args.limit]],
|
|
706
|
+
"total": len(hits),
|
|
707
|
+
"truncated": len(hits) > args.limit,
|
|
708
|
+
}
|
|
709
|
+
elif args.mode == "impact":
|
|
710
|
+
reached = await anyio.to_thread.run_sync(book.impact, name, args.distance, args.limit)
|
|
711
|
+
rings: dict[int, list[dict[str, Any]]] = {}
|
|
712
|
+
for depth, symbol in reached:
|
|
713
|
+
rings.setdefault(depth, []).append(symbol.as_value())
|
|
714
|
+
body |= {
|
|
715
|
+
"rings": [
|
|
716
|
+
{"distance": depth, "symbols": found} for depth, found in sorted(rings.items())
|
|
717
|
+
],
|
|
718
|
+
"total": len(reached),
|
|
719
|
+
}
|
|
720
|
+
else:
|
|
721
|
+
found, total = await anyio.to_thread.run_sync(
|
|
722
|
+
book.entities, args.path, args.kind, args.limit, args.offset
|
|
723
|
+
)
|
|
724
|
+
body |= {
|
|
725
|
+
"symbols": _symbols(found),
|
|
726
|
+
"total": total,
|
|
727
|
+
"truncated": args.offset + args.limit < total,
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
return {
|
|
731
|
+
"mode": args.mode,
|
|
732
|
+
"query": args.query,
|
|
733
|
+
"definitions": defined,
|
|
734
|
+
"offset": args.offset,
|
|
735
|
+
"indexed_files": indexed,
|
|
736
|
+
**body,
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
ctx.require(TOOLS).register(
|
|
740
|
+
define_tool(
|
|
741
|
+
"code_index",
|
|
742
|
+
INDEX_DESCRIPTION,
|
|
743
|
+
parameters=IndexArgs,
|
|
744
|
+
output=ToolOutput(schema=IndexValue, render=_render_index),
|
|
745
|
+
execute=index_tool,
|
|
746
|
+
# See `CodeGraphSeam` — the lock makes the outcome correct, and this
|
|
747
|
+
# keeps the scheduler from queueing two against each other (B6).
|
|
748
|
+
is_concurrency_safe=False,
|
|
749
|
+
self_limits=True,
|
|
750
|
+
**simple_views("search", "Index code", "paths"),
|
|
751
|
+
)
|
|
752
|
+
)
|
|
753
|
+
ctx.require(TOOLS).register(
|
|
754
|
+
define_tool(
|
|
755
|
+
"code_graph",
|
|
756
|
+
GRAPH_DESCRIPTION,
|
|
757
|
+
parameters=GraphArgs,
|
|
758
|
+
output=ToolOutput(schema=GraphValue, render=_render_graph),
|
|
759
|
+
execute=graph_tool,
|
|
760
|
+
is_concurrency_safe=True,
|
|
761
|
+
self_limits=True,
|
|
762
|
+
**simple_views("search", "Code graph", "query"),
|
|
763
|
+
)
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
async def install(argument: str, _invocation: CommandContext) -> str:
|
|
767
|
+
"""`/code-graph install|status` — make the grammars ready, on purpose.
|
|
768
|
+
|
|
769
|
+
A **command** rather than a tool, per the seam's own rule: a person asks
|
|
770
|
+
the harness to provision, and it costs no model turn. It matters less
|
|
771
|
+
here than for `text-index` — 26 languages are inside the wheel and only
|
|
772
|
+
the long tail fetches — but "is this ready" should have one answer per
|
|
773
|
+
plugin, asked the same way.
|
|
774
|
+
"""
|
|
775
|
+
verb = argument.strip().lower() or "status"
|
|
776
|
+
if verb not in ("install", "status"):
|
|
777
|
+
return f"/code-graph takes `install` or `status`, not {argument.strip()!r}."
|
|
778
|
+
wanted = [one for one in (config.languages or DEFAULT_LANGUAGES) if indexable(one)]
|
|
779
|
+
if verb == "status":
|
|
780
|
+
ready, missing = await anyio.to_thread.run_sync(readiness, wanted)
|
|
781
|
+
line = f"grammars under {seam.grammars}: {len(ready)} of {len(wanted)} ready"
|
|
782
|
+
return line + (f"; missing {', '.join(missing)}" if missing else "")
|
|
783
|
+
ready, missing = await anyio.to_thread.run_sync(ensure, wanted)
|
|
784
|
+
if missing:
|
|
785
|
+
return (
|
|
786
|
+
f"{len(ready)} of {len(wanted)} grammars ready; could not fetch "
|
|
787
|
+
f"{', '.join(missing)} — a language outside the wheel's bundled set comes "
|
|
788
|
+
"from GitHub, so this needs network the first time."
|
|
789
|
+
)
|
|
790
|
+
return f"all {count_of(len(ready), 'grammar')} ready under {seam.grammars}."
|
|
791
|
+
|
|
792
|
+
command = CommandDefinition(
|
|
793
|
+
name="code-graph",
|
|
794
|
+
summary="Make the tree-sitter grammars ready, or report whether they are.",
|
|
795
|
+
argument_hint="[install|status]",
|
|
796
|
+
run=install,
|
|
797
|
+
)
|
|
798
|
+
contribute_item(
|
|
799
|
+
ctx,
|
|
800
|
+
COMMANDS,
|
|
801
|
+
command,
|
|
802
|
+
label="code-graph command",
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
offer_skills(ctx)
|
|
806
|
+
contribute(ctx, Diagnostic(id="code-graph", title="Code graph", order=60, read=seam.report))
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _symbols(rows: list[SymbolRow]) -> list[dict[str, Any]]:
|
|
810
|
+
return [one.as_value() for one in rows]
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _fts(query: str) -> str:
|
|
814
|
+
"""A model's words as an FTS5 expression, with its syntax neutralised.
|
|
815
|
+
|
|
816
|
+
FTS5 reads `-`, `"`, `*`, `(`, `:` and `NEAR` as operators, so a query like
|
|
817
|
+
`read-before-edit` is a syntax error and `foo:bar` is a column filter that
|
|
818
|
+
matches nothing. A model writing prose did not mean either. Every term is
|
|
819
|
+
quoted and the set is OR-ed, which is what "find symbols about these words"
|
|
820
|
+
should do — and it cannot raise on the model's phrasing.
|
|
821
|
+
"""
|
|
822
|
+
terms = [one for one in query.replace('"', " ").split() if one]
|
|
823
|
+
if not terms:
|
|
824
|
+
return '""'
|
|
825
|
+
return " OR ".join(f'"{one}"' for one in terms)
|