sentinel-codegraph 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sentinel_codegraph-0.3.0/.gitignore +44 -0
- sentinel_codegraph-0.3.0/PKG-INFO +251 -0
- sentinel_codegraph-0.3.0/README.md +241 -0
- sentinel_codegraph-0.3.0/dist_probe.txt +1 -0
- sentinel_codegraph-0.3.0/examples/dummy_tree.py +239 -0
- sentinel_codegraph-0.3.0/probe_dir/f.txt +1 -0
- sentinel_codegraph-0.3.0/pyproject.toml +37 -0
- sentinel_codegraph-0.3.0/src/codegraph/__init__.py +5 -0
- sentinel_codegraph-0.3.0/src/codegraph/__main__.py +6 -0
- sentinel_codegraph-0.3.0/src/codegraph/cli.py +475 -0
- sentinel_codegraph-0.3.0/src/codegraph/config.py +55 -0
- sentinel_codegraph-0.3.0/src/codegraph/graph_store.py +455 -0
- sentinel_codegraph-0.3.0/src/codegraph/models.py +87 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/__init__.py +27 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/base.py +114 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/lang_go.py +524 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/lang_python.py +458 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/lang_typescript.py +554 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/links.py +314 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/queries.py +114 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/raw_core.py +66 -0
- sentinel_codegraph-0.3.0/src/codegraph/parser/rows.py +191 -0
- sentinel_codegraph-0.3.0/src/codegraph/pipeline.py +370 -0
- sentinel_codegraph-0.3.0/src/codegraph/query.py +125 -0
- sentinel_codegraph-0.3.0/src/codegraph/tree.py +230 -0
- sentinel_codegraph-0.3.0/src/codegraph/walk.py +114 -0
- sentinel_codegraph-0.3.0/tests/test_calls.py +345 -0
- sentinel_codegraph-0.3.0/tests/test_config.py +51 -0
- sentinel_codegraph-0.3.0/tests/test_index_e2e.py +141 -0
- sentinel_codegraph-0.3.0/tests/test_parser_emitters.py +401 -0
- sentinel_codegraph-0.3.0/tests/test_query.py +401 -0
- sentinel_codegraph-0.3.0/tests/test_tree.py +312 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.venv/
|
|
9
|
+
.uv-cache/
|
|
10
|
+
|
|
11
|
+
# LanceDB test dataset
|
|
12
|
+
packages/api/lance-db/
|
|
13
|
+
|
|
14
|
+
# Environments / secrets
|
|
15
|
+
.env
|
|
16
|
+
.env.local
|
|
17
|
+
.env.production
|
|
18
|
+
|
|
19
|
+
.env.*.local
|
|
20
|
+
**/sandbox.env
|
|
21
|
+
**/*.pem
|
|
22
|
+
|
|
23
|
+
# Node / frontend (apps/web, etc.)
|
|
24
|
+
node_modules/
|
|
25
|
+
dist/
|
|
26
|
+
.output/
|
|
27
|
+
.vite/
|
|
28
|
+
.turbo/
|
|
29
|
+
*.tsbuildinfo
|
|
30
|
+
next-env.d.ts
|
|
31
|
+
|
|
32
|
+
# IDE / OS
|
|
33
|
+
.vscode/
|
|
34
|
+
.idea/
|
|
35
|
+
.DS_Store
|
|
36
|
+
Thumbs.db
|
|
37
|
+
*.swp
|
|
38
|
+
|
|
39
|
+
# Eval harness
|
|
40
|
+
packages/evals/results/
|
|
41
|
+
packages/evals/report/
|
|
42
|
+
packages/evals/sandbox/sentinel-workspace/
|
|
43
|
+
packages/evals/sandbox/tmp/
|
|
44
|
+
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sentinel-codegraph
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Sentinel code graph CLI: tree-sitter nodes/edges into SQL
|
|
5
|
+
Author: Sentinel
|
|
6
|
+
Requires-Python: >=3.13
|
|
7
|
+
Requires-Dist: ladybug==0.19.1
|
|
8
|
+
Requires-Dist: tree-sitter-language-pack>=1.20.0
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# Sentinel Code Graph CLI
|
|
12
|
+
|
|
13
|
+
Async CLI that scans a directory or file, extracts structural nodes
|
|
14
|
+
(files, classes, functions, methods, interfaces, types, imports) and
|
|
15
|
+
edges (contains, imports, calls) with raw tree-sitter, and stores them
|
|
16
|
+
in Ladybug (embedded property graph, zero setup).
|
|
17
|
+
|
|
18
|
+
- Default database is a known-location Ladybug file
|
|
19
|
+
(`~/.codegraph/graph.lbdb`), so query time never guesses where the
|
|
20
|
+
index lives. Pass `--db` to any command to override it.
|
|
21
|
+
- `--db :memory:` indexes ephemerally (lost when the process exits).
|
|
22
|
+
- Scope: **Python, TypeScript/JavaScript, Go**. (`queries.py` stays on
|
|
23
|
+
disk, unwired.)
|
|
24
|
+
- One database holds exactly one index: `--overwrite` flushes the
|
|
25
|
+
whole db first, so re-indexing is idempotent.
|
|
26
|
+
- `--no-persist` dry-runs the pipeline (parse → nodes → edges → print,
|
|
27
|
+
no DB writes).
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
From the repo root (workspace member, latest pinned deps in `uv.lock`):
|
|
32
|
+
|
|
33
|
+
```powershell
|
|
34
|
+
uv sync
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
```powershell
|
|
40
|
+
# index a tree into the default DB (re-runnable; --overwrite flushes first)
|
|
41
|
+
uv run --package codegraph python -m codegraph.cli index ./packages/api/src --overwrite
|
|
42
|
+
|
|
43
|
+
# index a single file (root = its parent dir; must be a supported language)
|
|
44
|
+
uv run --package codegraph python -m codegraph.cli index ./packages/api/main.py
|
|
45
|
+
|
|
46
|
+
# inspect the database
|
|
47
|
+
uv run --package codegraph python -m codegraph.cli stats
|
|
48
|
+
|
|
49
|
+
# index and print the hierarchy tree of the indexed root
|
|
50
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output tree
|
|
51
|
+
|
|
52
|
+
# dump every collected node (kind, lines, parent, children, callees)
|
|
53
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output nodes
|
|
54
|
+
|
|
55
|
+
# list every calls edge (caller -> callee, implementation order)
|
|
56
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output calls
|
|
57
|
+
|
|
58
|
+
# dry-run: print without persisting
|
|
59
|
+
uv run --package codegraph python -m codegraph.cli index ./src --no-persist --output tree
|
|
60
|
+
|
|
61
|
+
# Ephemeral index (no file written)
|
|
62
|
+
uv run --package codegraph python -m codegraph.cli index ./src --db :memory: --output tree
|
|
63
|
+
|
|
64
|
+
# explore the index (read-only; works from anywhere — default DB is known)
|
|
65
|
+
uv run --package codegraph python -m codegraph.cli query overview
|
|
66
|
+
uv run --package codegraph python -m codegraph.cli query files
|
|
67
|
+
uv run --package codegraph python -m codegraph.cli query search --name reviewWorkflowV2
|
|
68
|
+
uv run --package codegraph python -m codegraph.cli query callees --name reviewWorkflowV2 --json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Flow
|
|
72
|
+
|
|
73
|
+
`amain` parses args, then three linear stages in `pipeline.py`:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
read(target) -> build_graph(root, items) -> out(db, root, graph, ...)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
- **read** — discover (`walk`: suffix → language, prune noise dirs,
|
|
80
|
+
skip oversize/undecodable) + read off the loop → `ScannedSources(root,
|
|
81
|
+
items)`. I/O.
|
|
82
|
+
- **build_graph** — pure, two passes: (1) collect — `parse_file_input`
|
|
83
|
+
per file (blank source or unsupported language → skip, counted)
|
|
84
|
+
into nodes + `contains` / `imports` plus buffered call sites; (2)
|
|
85
|
+
link — `resolve_call_edges` joins sites against the `(file, name)`
|
|
86
|
+
definition registry + per-file import maps (alias-aware, joined on
|
|
87
|
+
a single candidate: the defining-module original when the import
|
|
88
|
+
carries one, else the bound name; Go dot-imported files are a
|
|
89
|
+
fallback) into `calls` with real node ids, dropping unresolvable
|
|
90
|
+
sites. Merge → `BuiltGraph(root, files, skipped, nodes, edges)`.
|
|
91
|
+
No I/O, no DB.
|
|
92
|
+
- **out** — persist (`create_all`, `clear_all` when `--overwrite`,
|
|
93
|
+
`add_all`) then print (`summary` | `tree` | `nodes` | `calls`) →
|
|
94
|
+
`IndexResult`. I/O.
|
|
95
|
+
|
|
96
|
+
`cli.py` holds only arg parsing + `amain` wiring + `run_stats`;
|
|
97
|
+
`__main__.py` calls `cli.main()`.
|
|
98
|
+
|
|
99
|
+
## Python emitter
|
|
100
|
+
|
|
101
|
+
Each Python file is walked once (`collect_python_file`):
|
|
102
|
+
`Node` + `Edge(CONTAINS)` with `base:name:start:end` ids for defs,
|
|
103
|
+
`Node(IMPORT)` + `Edge(IMPORTS)` per imported name (keeping the
|
|
104
|
+
`as`-alias original, e.g. `from utils import helper as h` records bound
|
|
105
|
+
`h` + original `helper`), and bare-name calls buffered unresolved with
|
|
106
|
+
their call-site line. The link phase then joins each site — same-file
|
|
107
|
+
hit first, else the import map's resolved file + original name — into
|
|
108
|
+
`calls` edges with real node ids. Absolute imports tolerate a
|
|
109
|
+
root-relative prefix on indexed paths (`src/app/…` satisfies
|
|
110
|
+
`import app.…`). Builtins, stdlib, third-party, star-import calls,
|
|
111
|
+
attribute calls (`obj.method(…)`, `self.x(…)`), and module-level call
|
|
112
|
+
sites yield no edges. Decorator applications (`@retry`,
|
|
113
|
+
`@with_logging(…)`) buffer as call sites on the decorated def/class
|
|
114
|
+
with the @-line and resolve like ordinary calls (attribute decorators
|
|
115
|
+
such as `@app.get(…)` are skipped). One edge per caller → callee pair,
|
|
116
|
+
each stamped with its call-site line. Calls render in implementation
|
|
117
|
+
order, not alphabetical.
|
|
118
|
+
|
|
119
|
+
## TypeScript / JavaScript collector
|
|
120
|
+
|
|
121
|
+
Each TS/JS file is walked once (`collect_ts_file`): classes,
|
|
122
|
+
functions, methods, interfaces, type aliases, and arrow-bound consts
|
|
123
|
+
(`export` wrappers are transparent) → `Node` + `Edge(CONTAINS)`;
|
|
124
|
+
imports keep alias originals (`import {a as b}` → bound `b` +
|
|
125
|
+
original `a`; default imports join on the bound name); bare calls and
|
|
126
|
+
`new C()` constructions buffer unresolved. Relative specifiers resolve
|
|
127
|
+
with extension + `/index` fallbacks; bare (npm) specifiers never
|
|
128
|
+
resolve. Member calls (`obj.m()`), builtins, and module-level sites
|
|
129
|
+
yield no edges.
|
|
130
|
+
|
|
131
|
+
## Go collector
|
|
132
|
+
|
|
133
|
+
Each Go file is walked once (`collect_go_file`): funcs, methods
|
|
134
|
+
(reparented to their receiver struct when same-file), struct types →
|
|
135
|
+
`class`, interface types → `interface` (with `method_elem` children as
|
|
136
|
+
methods), other named types → `type`. Blank imports bind `*`, dot
|
|
137
|
+
imports bind `.` (their target files are searched for otherwise
|
|
138
|
+
unresolved bare sites). Import tails match indexed `.go` stems;
|
|
139
|
+
selector calls (`pkg.Fn()`), builtins, and package-level sites yield
|
|
140
|
+
no edges.
|
|
141
|
+
|
|
142
|
+
## Schema (Ladybug)
|
|
143
|
+
|
|
144
|
+
- `CodeNode`: `id (PK), root, file_path, kind (file|class|function|method|interface|type|import),
|
|
145
|
+
name, language, start_line, end_line, parent_id, is_placeholder`
|
|
146
|
+
- `Contains` / `Imports` / `Calls`: rels between `CodeNode` rows
|
|
147
|
+
(`target_module` on `Imports`, `site_line` on `Calls`).
|
|
148
|
+
|
|
149
|
+
`site_line` is the 1-based call-site line inside the caller (`Calls`
|
|
150
|
+
rels only; `NULL` = unknown). Callees sort by it.
|
|
151
|
+
|
|
152
|
+
## Node ids (all languages)
|
|
153
|
+
|
|
154
|
+
- file node: `base` (path anchored at the CLI root, e.g. `workflows/review.py`)
|
|
155
|
+
- def node: `base:name:start:end` (e.g. `workflows/review.py:helper:10:15`)
|
|
156
|
+
- import node: `base:import:<name>:<line>`
|
|
157
|
+
|
|
158
|
+
Every stored `Calls` edge points at real node ids — unresolvable call
|
|
159
|
+
sites are dropped at build time, so no placeholder rows exist
|
|
160
|
+
(`is_placeholder` stays on the schema for old databases only).
|
|
161
|
+
Querier's check:
|
|
162
|
+
|
|
163
|
+
```cypher
|
|
164
|
+
// 1. file's nodes (find the caller)
|
|
165
|
+
MATCH (f:CodeNode {id: '<file>'})-[:Contains]->(n) RETURN n;
|
|
166
|
+
// 2. caller's callees, in implementation order
|
|
167
|
+
MATCH (c:CodeNode {id: '<caller id>'})-[e:Calls]->(d) RETURN d ORDER BY e.site_line;
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Tables are created with `create_all` (`IF NOT EXISTS`): after a
|
|
171
|
+
schema change, delete the `.lbdb` file and re-index.
|
|
172
|
+
|
|
173
|
+
## Querying (agent ladder)
|
|
174
|
+
|
|
175
|
+
The `query` verbs are read-only and built for agents that don't know
|
|
176
|
+
node id shapes — every step returns the ids the next step needs:
|
|
177
|
+
|
|
178
|
+
1. `query overview` — what's indexed (counts by kind/edge/language).
|
|
179
|
+
2. `query files` — rel paths (a file node id *is* its rel path).
|
|
180
|
+
3. `query search --name <fragment> [--kind …] [--file …] [--limit N]` —
|
|
181
|
+
substring-match def names → full rows with ids. Zero hits and
|
|
182
|
+
truncation (`truncated: true`) tell the agent to rephrase/narrow.
|
|
183
|
+
4. `query node|callees|callers|children --id <id>` (or `--name` exact +
|
|
184
|
+
optional `--file`; ambiguity is an error listing hits) and
|
|
185
|
+
`query imports --file <rel>` — drill down; new names chain back
|
|
186
|
+
to `search`.
|
|
187
|
+
|
|
188
|
+
`--root` narrows to one indexed root (default: all), `--json` emits
|
|
189
|
+
the stable chaining envelope
|
|
190
|
+
(`{root, verb, count, truncated, items[]}` with
|
|
191
|
+
`{id, kind, name, file_path, language, start_line, end_line,
|
|
192
|
+
parent_id}` plus `site_line` on calls items and `target_module` on
|
|
193
|
+
imports items). Querying a database that was never indexed exits 1
|
|
194
|
+
with a hint instead of an empty result — and read paths never create
|
|
195
|
+
files or directories (only `index` persistence creates
|
|
196
|
+
`~/.codegraph/`).
|
|
197
|
+
|
|
198
|
+
## Module map
|
|
199
|
+
|
|
200
|
+
- `cli.py` — arg parsing, `amain` wiring (`read → build_graph → out`), `main`, `run_stats`, `run_query`
|
|
201
|
+
- `query.py` — pure exploration shaping: `search_nodes`, `exact_matches`, JSON envelope, human rendering
|
|
202
|
+
- `pipeline.py` — `read` / `build_graph` / `out`, `ScannedSources` / `BuiltGraph` / `IndexResult`, `OutputMode`, `print_tree` / `print_nodes` / `print_calls`
|
|
203
|
+
- `parser/__init__.py` — barrel: re-exports only, no logic
|
|
204
|
+
- `parser/rows.py` — `FileRows`, `build_file_rows`, `build_python_file_rows`
|
|
205
|
+
- `parser/links.py` — `ImportEntry`, `build_file_import_map`, `build_dot_import_targets`, `resolve_call_edges` (+ per-language module-specifier resolvers)
|
|
206
|
+
- `parser/lang_python.py` — Python collect phase (`collect_python_file`)
|
|
207
|
+
- `parser/lang_typescript.py` — TS/JS collect phase (`collect_ts_file`)
|
|
208
|
+
- `parser/lang_go.py` — Go collect phase (`collect_go_file`)
|
|
209
|
+
- `parser/queries.py` — kept, unwired
|
|
210
|
+
- `parser/base.py` — `ParsedFile` / `ParsedDefinition` / `ParsedImport` / `ParsedCall` IR types
|
|
211
|
+
- `parser/raw_core.py` — tree-sitter `parse`, `walk`, `span`, text helpers
|
|
212
|
+
- `models.py` — `Node` / `Edge` dataclasses (+ `NodeKind`, `EdgeKind`)
|
|
213
|
+
- `graph_store.py` — `LadybugStore` (UNWIND ingest, `clear_all`, `add_all`, count/list queries, narrow `get_node` / `callees` / `callers` / `children` / `file_imports` / `list_files` reads)
|
|
214
|
+
- `tree.py` — `GraphSnapshot`, `render_tree` (nesting by `contains`, calls in `site_line` order)
|
|
215
|
+
- `walk.py` — suffix → language discovery, noise-dir pruning
|
|
216
|
+
- `config.py` — `DEFAULT_DB` (`~/.codegraph/graph.lbdb`) + `--db` path resolution: file path or `:memory:` (`ResolvedDb`)
|
|
217
|
+
|
|
218
|
+
## Design notes
|
|
219
|
+
|
|
220
|
+
- Pipeline is functional: `read -> build_graph -> out`. Extractors and
|
|
221
|
+
builders are pure; I/O lives only at the edge (discover + read,
|
|
222
|
+
persist, print).
|
|
223
|
+
- Build output is a simple flat graph: `BuiltGraph` carries merged
|
|
224
|
+
`nodes` + `edges` tuples only. Every consumer (persist, snapshot,
|
|
225
|
+
printers) derives what it needs from those two lists.
|
|
226
|
+
- Graph values are frozen dataclasses holding tuples — immutable
|
|
227
|
+
snapshots, safe to share between build and out.
|
|
228
|
+
- Parsing uses raw `tree_sitter_language_pack.get_parser().parse()`.
|
|
229
|
+
Import *names* are recovered from the statement source text in pure
|
|
230
|
+
Python.
|
|
231
|
+
- Parsing runs on the event-loop thread (tree-sitter objects are not
|
|
232
|
+
thread-safe); only file I/O goes through `asyncio.to_thread`.
|
|
233
|
+
- Ladybug is embedded: one `AsyncConnection` per store, and `out()`
|
|
234
|
+
holds a single store from persist through the snapshot reads, so
|
|
235
|
+
`:memory:` databases stay alive for the whole call.
|
|
236
|
+
- Calls are bare-name only (`name(…)`), alias-aware via the import
|
|
237
|
+
map (`from utils import helper as h` + `h()` resolves to `helper`;
|
|
238
|
+
TS `import {a as b}` + `b()` resolves to `a`; TS default imports
|
|
239
|
+
join on the bound name; Go dot-imported files are a fallback);
|
|
240
|
+
`new C()` in TS counts as a call. Attribute / member calls
|
|
241
|
+
(`obj.method()`, `self.x()`, `pkg.Fn()`), builtins, stdlib,
|
|
242
|
+
third-party, star-import calls, and module-level call sites yield
|
|
243
|
+
no edges.
|
|
244
|
+
- Files with unsupported suffixes are discovered, then skipped
|
|
245
|
+
(`(N skipped)` in the summary).
|
|
246
|
+
|
|
247
|
+
## Tests
|
|
248
|
+
|
|
249
|
+
```powershell
|
|
250
|
+
uv run --package codegraph pytest tests
|
|
251
|
+
```
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# Sentinel Code Graph CLI
|
|
2
|
+
|
|
3
|
+
Async CLI that scans a directory or file, extracts structural nodes
|
|
4
|
+
(files, classes, functions, methods, interfaces, types, imports) and
|
|
5
|
+
edges (contains, imports, calls) with raw tree-sitter, and stores them
|
|
6
|
+
in Ladybug (embedded property graph, zero setup).
|
|
7
|
+
|
|
8
|
+
- Default database is a known-location Ladybug file
|
|
9
|
+
(`~/.codegraph/graph.lbdb`), so query time never guesses where the
|
|
10
|
+
index lives. Pass `--db` to any command to override it.
|
|
11
|
+
- `--db :memory:` indexes ephemerally (lost when the process exits).
|
|
12
|
+
- Scope: **Python, TypeScript/JavaScript, Go**. (`queries.py` stays on
|
|
13
|
+
disk, unwired.)
|
|
14
|
+
- One database holds exactly one index: `--overwrite` flushes the
|
|
15
|
+
whole db first, so re-indexing is idempotent.
|
|
16
|
+
- `--no-persist` dry-runs the pipeline (parse → nodes → edges → print,
|
|
17
|
+
no DB writes).
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
From the repo root (workspace member, latest pinned deps in `uv.lock`):
|
|
22
|
+
|
|
23
|
+
```powershell
|
|
24
|
+
uv sync
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Usage
|
|
28
|
+
|
|
29
|
+
```powershell
|
|
30
|
+
# index a tree into the default DB (re-runnable; --overwrite flushes first)
|
|
31
|
+
uv run --package codegraph python -m codegraph.cli index ./packages/api/src --overwrite
|
|
32
|
+
|
|
33
|
+
# index a single file (root = its parent dir; must be a supported language)
|
|
34
|
+
uv run --package codegraph python -m codegraph.cli index ./packages/api/main.py
|
|
35
|
+
|
|
36
|
+
# inspect the database
|
|
37
|
+
uv run --package codegraph python -m codegraph.cli stats
|
|
38
|
+
|
|
39
|
+
# index and print the hierarchy tree of the indexed root
|
|
40
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output tree
|
|
41
|
+
|
|
42
|
+
# dump every collected node (kind, lines, parent, children, callees)
|
|
43
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output nodes
|
|
44
|
+
|
|
45
|
+
# list every calls edge (caller -> callee, implementation order)
|
|
46
|
+
uv run --package codegraph python -m codegraph.cli index ./src --output calls
|
|
47
|
+
|
|
48
|
+
# dry-run: print without persisting
|
|
49
|
+
uv run --package codegraph python -m codegraph.cli index ./src --no-persist --output tree
|
|
50
|
+
|
|
51
|
+
# Ephemeral index (no file written)
|
|
52
|
+
uv run --package codegraph python -m codegraph.cli index ./src --db :memory: --output tree
|
|
53
|
+
|
|
54
|
+
# explore the index (read-only; works from anywhere — default DB is known)
|
|
55
|
+
uv run --package codegraph python -m codegraph.cli query overview
|
|
56
|
+
uv run --package codegraph python -m codegraph.cli query files
|
|
57
|
+
uv run --package codegraph python -m codegraph.cli query search --name reviewWorkflowV2
|
|
58
|
+
uv run --package codegraph python -m codegraph.cli query callees --name reviewWorkflowV2 --json
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Flow
|
|
62
|
+
|
|
63
|
+
`amain` parses args, then three linear stages in `pipeline.py`:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
read(target) -> build_graph(root, items) -> out(db, root, graph, ...)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
- **read** — discover (`walk`: suffix → language, prune noise dirs,
|
|
70
|
+
skip oversize/undecodable) + read off the loop → `ScannedSources(root,
|
|
71
|
+
items)`. I/O.
|
|
72
|
+
- **build_graph** — pure, two passes: (1) collect — `parse_file_input`
|
|
73
|
+
per file (blank source or unsupported language → skip, counted)
|
|
74
|
+
into nodes + `contains` / `imports` plus buffered call sites; (2)
|
|
75
|
+
link — `resolve_call_edges` joins sites against the `(file, name)`
|
|
76
|
+
definition registry + per-file import maps (alias-aware, joined on
|
|
77
|
+
a single candidate: the defining-module original when the import
|
|
78
|
+
carries one, else the bound name; Go dot-imported files are a
|
|
79
|
+
fallback) into `calls` with real node ids, dropping unresolvable
|
|
80
|
+
sites. Merge → `BuiltGraph(root, files, skipped, nodes, edges)`.
|
|
81
|
+
No I/O, no DB.
|
|
82
|
+
- **out** — persist (`create_all`, `clear_all` when `--overwrite`,
|
|
83
|
+
`add_all`) then print (`summary` | `tree` | `nodes` | `calls`) →
|
|
84
|
+
`IndexResult`. I/O.
|
|
85
|
+
|
|
86
|
+
`cli.py` holds only arg parsing + `amain` wiring + `run_stats`;
|
|
87
|
+
`__main__.py` calls `cli.main()`.
|
|
88
|
+
|
|
89
|
+
## Python emitter
|
|
90
|
+
|
|
91
|
+
Each Python file is walked once (`collect_python_file`):
|
|
92
|
+
`Node` + `Edge(CONTAINS)` with `base:name:start:end` ids for defs,
|
|
93
|
+
`Node(IMPORT)` + `Edge(IMPORTS)` per imported name (keeping the
|
|
94
|
+
`as`-alias original, e.g. `from utils import helper as h` records bound
|
|
95
|
+
`h` + original `helper`), and bare-name calls buffered unresolved with
|
|
96
|
+
their call-site line. The link phase then joins each site — same-file
|
|
97
|
+
hit first, else the import map's resolved file + original name — into
|
|
98
|
+
`calls` edges with real node ids. Absolute imports tolerate a
|
|
99
|
+
root-relative prefix on indexed paths (`src/app/…` satisfies
|
|
100
|
+
`import app.…`). Builtins, stdlib, third-party, star-import calls,
|
|
101
|
+
attribute calls (`obj.method(…)`, `self.x(…)`), and module-level call
|
|
102
|
+
sites yield no edges. Decorator applications (`@retry`,
|
|
103
|
+
`@with_logging(…)`) buffer as call sites on the decorated def/class
|
|
104
|
+
with the @-line and resolve like ordinary calls (attribute decorators
|
|
105
|
+
such as `@app.get(…)` are skipped). One edge per caller → callee pair,
|
|
106
|
+
each stamped with its call-site line. Calls render in implementation
|
|
107
|
+
order, not alphabetical.
|
|
108
|
+
|
|
109
|
+
## TypeScript / JavaScript collector
|
|
110
|
+
|
|
111
|
+
Each TS/JS file is walked once (`collect_ts_file`): classes,
|
|
112
|
+
functions, methods, interfaces, type aliases, and arrow-bound consts
|
|
113
|
+
(`export` wrappers are transparent) → `Node` + `Edge(CONTAINS)`;
|
|
114
|
+
imports keep alias originals (`import {a as b}` → bound `b` +
|
|
115
|
+
original `a`; default imports join on the bound name); bare calls and
|
|
116
|
+
`new C()` constructions buffer unresolved. Relative specifiers resolve
|
|
117
|
+
with extension + `/index` fallbacks; bare (npm) specifiers never
|
|
118
|
+
resolve. Member calls (`obj.m()`), builtins, and module-level sites
|
|
119
|
+
yield no edges.
|
|
120
|
+
|
|
121
|
+
## Go collector
|
|
122
|
+
|
|
123
|
+
Each Go file is walked once (`collect_go_file`): funcs, methods
|
|
124
|
+
(reparented to their receiver struct when same-file), struct types →
|
|
125
|
+
`class`, interface types → `interface` (with `method_elem` children as
|
|
126
|
+
methods), other named types → `type`. Blank imports bind `*`, dot
|
|
127
|
+
imports bind `.` (their target files are searched for otherwise
|
|
128
|
+
unresolved bare sites). Import tails match indexed `.go` stems;
|
|
129
|
+
selector calls (`pkg.Fn()`), builtins, and package-level sites yield
|
|
130
|
+
no edges.
|
|
131
|
+
|
|
132
|
+
## Schema (Ladybug)
|
|
133
|
+
|
|
134
|
+
- `CodeNode`: `id (PK), root, file_path, kind (file|class|function|method|interface|type|import),
|
|
135
|
+
name, language, start_line, end_line, parent_id, is_placeholder`
|
|
136
|
+
- `Contains` / `Imports` / `Calls`: rels between `CodeNode` rows
|
|
137
|
+
(`target_module` on `Imports`, `site_line` on `Calls`).
|
|
138
|
+
|
|
139
|
+
`site_line` is the 1-based call-site line inside the caller (`Calls`
|
|
140
|
+
rels only; `NULL` = unknown). Callees sort by it.
|
|
141
|
+
|
|
142
|
+
## Node ids (all languages)
|
|
143
|
+
|
|
144
|
+
- file node: `base` (path anchored at the CLI root, e.g. `workflows/review.py`)
|
|
145
|
+
- def node: `base:name:start:end` (e.g. `workflows/review.py:helper:10:15`)
|
|
146
|
+
- import node: `base:import:<name>:<line>`
|
|
147
|
+
|
|
148
|
+
Every stored `Calls` edge points at real node ids — unresolvable call
|
|
149
|
+
sites are dropped at build time, so no placeholder rows exist
|
|
150
|
+
(`is_placeholder` stays on the schema for old databases only).
|
|
151
|
+
Querier's check:
|
|
152
|
+
|
|
153
|
+
```cypher
|
|
154
|
+
// 1. file's nodes (find the caller)
|
|
155
|
+
MATCH (f:CodeNode {id: '<file>'})-[:Contains]->(n) RETURN n;
|
|
156
|
+
// 2. caller's callees, in implementation order
|
|
157
|
+
MATCH (c:CodeNode {id: '<caller id>'})-[e:Calls]->(d) RETURN d ORDER BY e.site_line;
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Tables are created with `create_all` (`IF NOT EXISTS`): after a
|
|
161
|
+
schema change, delete the `.lbdb` file and re-index.
|
|
162
|
+
|
|
163
|
+
## Querying (agent ladder)
|
|
164
|
+
|
|
165
|
+
The `query` verbs are read-only and built for agents that don't know
|
|
166
|
+
node id shapes — every step returns the ids the next step needs:
|
|
167
|
+
|
|
168
|
+
1. `query overview` — what's indexed (counts by kind/edge/language).
|
|
169
|
+
2. `query files` — rel paths (a file node id *is* its rel path).
|
|
170
|
+
3. `query search --name <fragment> [--kind …] [--file …] [--limit N]` —
|
|
171
|
+
substring-match def names → full rows with ids. Zero hits and
|
|
172
|
+
truncation (`truncated: true`) tell the agent to rephrase/narrow.
|
|
173
|
+
4. `query node|callees|callers|children --id <id>` (or `--name` exact +
|
|
174
|
+
optional `--file`; ambiguity is an error listing hits) and
|
|
175
|
+
`query imports --file <rel>` — drill down; new names chain back
|
|
176
|
+
to `search`.
|
|
177
|
+
|
|
178
|
+
`--root` narrows to one indexed root (default: all), `--json` emits
|
|
179
|
+
the stable chaining envelope
|
|
180
|
+
(`{root, verb, count, truncated, items[]}` with
|
|
181
|
+
`{id, kind, name, file_path, language, start_line, end_line,
|
|
182
|
+
parent_id}` plus `site_line` on calls items and `target_module` on
|
|
183
|
+
imports items). Querying a database that was never indexed exits 1
|
|
184
|
+
with a hint instead of an empty result — and read paths never create
|
|
185
|
+
files or directories (only `index` persistence creates
|
|
186
|
+
`~/.codegraph/`).
|
|
187
|
+
|
|
188
|
+
## Module map
|
|
189
|
+
|
|
190
|
+
- `cli.py` — arg parsing, `amain` wiring (`read → build_graph → out`), `main`, `run_stats`, `run_query`
|
|
191
|
+
- `query.py` — pure exploration shaping: `search_nodes`, `exact_matches`, JSON envelope, human rendering
|
|
192
|
+
- `pipeline.py` — `read` / `build_graph` / `out`, `ScannedSources` / `BuiltGraph` / `IndexResult`, `OutputMode`, `print_tree` / `print_nodes` / `print_calls`
|
|
193
|
+
- `parser/__init__.py` — barrel: re-exports only, no logic
|
|
194
|
+
- `parser/rows.py` — `FileRows`, `build_file_rows`, `build_python_file_rows`
|
|
195
|
+
- `parser/links.py` — `ImportEntry`, `build_file_import_map`, `build_dot_import_targets`, `resolve_call_edges` (+ per-language module-specifier resolvers)
|
|
196
|
+
- `parser/lang_python.py` — Python collect phase (`collect_python_file`)
|
|
197
|
+
- `parser/lang_typescript.py` — TS/JS collect phase (`collect_ts_file`)
|
|
198
|
+
- `parser/lang_go.py` — Go collect phase (`collect_go_file`)
|
|
199
|
+
- `parser/queries.py` — kept, unwired
|
|
200
|
+
- `parser/base.py` — `ParsedFile` / `ParsedDefinition` / `ParsedImport` / `ParsedCall` IR types
|
|
201
|
+
- `parser/raw_core.py` — tree-sitter `parse`, `walk`, `span`, text helpers
|
|
202
|
+
- `models.py` — `Node` / `Edge` dataclasses (+ `NodeKind`, `EdgeKind`)
|
|
203
|
+
- `graph_store.py` — `LadybugStore` (UNWIND ingest, `clear_all`, `add_all`, count/list queries, narrow `get_node` / `callees` / `callers` / `children` / `file_imports` / `list_files` reads)
|
|
204
|
+
- `tree.py` — `GraphSnapshot`, `render_tree` (nesting by `contains`, calls in `site_line` order)
|
|
205
|
+
- `walk.py` — suffix → language discovery, noise-dir pruning
|
|
206
|
+
- `config.py` — `DEFAULT_DB` (`~/.codegraph/graph.lbdb`) + `--db` path resolution: file path or `:memory:` (`ResolvedDb`)
|
|
207
|
+
|
|
208
|
+
## Design notes
|
|
209
|
+
|
|
210
|
+
- Pipeline is functional: `read -> build_graph -> out`. Extractors and
|
|
211
|
+
builders are pure; I/O lives only at the edge (discover + read,
|
|
212
|
+
persist, print).
|
|
213
|
+
- Build output is a simple flat graph: `BuiltGraph` carries merged
|
|
214
|
+
`nodes` + `edges` tuples only. Every consumer (persist, snapshot,
|
|
215
|
+
printers) derives what it needs from those two lists.
|
|
216
|
+
- Graph values are frozen dataclasses holding tuples — immutable
|
|
217
|
+
snapshots, safe to share between build and out.
|
|
218
|
+
- Parsing uses raw `tree_sitter_language_pack.get_parser().parse()`.
|
|
219
|
+
Import *names* are recovered from the statement source text in pure
|
|
220
|
+
Python.
|
|
221
|
+
- Parsing runs on the event-loop thread (tree-sitter objects are not
|
|
222
|
+
thread-safe); only file I/O goes through `asyncio.to_thread`.
|
|
223
|
+
- Ladybug is embedded: one `AsyncConnection` per store, and `out()`
|
|
224
|
+
holds a single store from persist through the snapshot reads, so
|
|
225
|
+
`:memory:` databases stay alive for the whole call.
|
|
226
|
+
- Calls are bare-name only (`name(…)`), alias-aware via the import
|
|
227
|
+
map (`from utils import helper as h` + `h()` resolves to `helper`;
|
|
228
|
+
TS `import {a as b}` + `b()` resolves to `a`; TS default imports
|
|
229
|
+
join on the bound name; Go dot-imported files are a fallback);
|
|
230
|
+
`new C()` in TS counts as a call. Attribute / member calls
|
|
231
|
+
(`obj.method()`, `self.x()`, `pkg.Fn()`), builtins, stdlib,
|
|
232
|
+
third-party, star-import calls, and module-level call sites yield
|
|
233
|
+
no edges.
|
|
234
|
+
- Files with unsupported suffixes are discovered, then skipped
|
|
235
|
+
(`(N skipped)` in the summary).
|
|
236
|
+
|
|
237
|
+
## Tests
|
|
238
|
+
|
|
239
|
+
```powershell
|
|
240
|
+
uv run --package codegraph pytest tests
|
|
241
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
marker-content
|