scip-cli 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scip_cli-1.2.0/scip_cli.egg-info → scip_cli-1.3.0}/PKG-INFO +29 -15
- {scip_cli-1.2.0 → scip_cli-1.3.0}/README.md +28 -14
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/SKILL.md +3 -3
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/__init__.py +1 -1
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/__main__.py +8 -2
- scip_cli-1.3.0/scip_cli/cache.py +70 -0
- scip_cli-1.3.0/scip_cli/commands/reindex.py +55 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/search.py +1 -1
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/constants.py +7 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/indexing.py +124 -46
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/merge.py +5 -4
- scip_cli-1.3.0/scip_cli/scope.py +74 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0/scip_cli.egg-info}/PKG-INFO +29 -15
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli.egg-info/SOURCES.txt +2 -0
- scip_cli-1.3.0/tests/test_cache.py +54 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_merge.py +72 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_pure_functions.py +22 -1
- scip_cli-1.3.0/tests/test_scope.py +61 -0
- scip_cli-1.2.0/scip_cli/cache.py +0 -34
- scip_cli-1.2.0/scip_cli/commands/reindex.py +0 -28
- scip_cli-1.2.0/tests/test_cache.py +0 -22
- {scip_cli-1.2.0 → scip_cli-1.3.0}/LICENSE +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/MANIFEST.in +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/pyproject.toml +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/cli_args.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/__init__.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/def_cmd.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/members.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/rdeps.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/refs.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/skill.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/commands/symbols.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/config.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/discover.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/lib.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/output.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/paths.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/project.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/queries.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/scip_tool.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/session.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/source.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/sql.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli/symbols.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli.egg-info/dependency_links.txt +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli.egg-info/entry_points.txt +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/scip_cli.egg-info/top_level.txt +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/setup.cfg +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/setup.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_composability.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_config.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_discover.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_indexer_env.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_scip_tool.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_smoke_cli.py +0 -0
- {scip_cli-1.2.0 → scip_cli-1.3.0}/tests/test_typescript_projects.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scip-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Fast code intelligence via SCIP indexes
|
|
5
5
|
Home-page: https://github.com/flesler/scip-cli
|
|
6
6
|
Author: Ariel Flesler
|
|
@@ -79,18 +79,21 @@ python -m scip_cli --help
|
|
|
79
79
|
|
|
80
80
|
### 2. Install prerequisites (optional)
|
|
81
81
|
|
|
82
|
-
scip-cli
|
|
82
|
+
On **first index**, scip-cli runs language indexers and builds a SQLite cache. You can let it fetch tools on demand, or install them globally ahead of time so the first run does not download via `npx`:
|
|
83
83
|
|
|
84
84
|
**Option A: Zero extra setup (recommended)**
|
|
85
85
|
|
|
86
86
|
Install `scip-cli` and run it. On first index, scip-cli will:
|
|
87
87
|
|
|
88
|
-
- Download `scip-typescript` / `scip-python` via `npx` when
|
|
88
|
+
- Download `scip-typescript` / `scip-python` via `npx` when not already on PATH
|
|
89
89
|
- Download the `scip` converter binary from [GitHub releases](https://github.com/scip-code/scip/releases) into `~/.cache/scip-cli/bin/` when not already on PATH
|
|
90
|
+
- Walk the repo for `tsconfig*.json` project roots (TypeScript monorepos), run `scip-typescript` per project (parallel by default), convert each partial index, then merge into one `index.db`
|
|
90
91
|
|
|
91
|
-
No `.scip-cli.json` required
|
|
92
|
+
No `.scip-cli.json` required for discovery. Subsequent queries read the cached database only.
|
|
92
93
|
|
|
93
|
-
**Option B: Install globally
|
|
94
|
+
**Option B: Install indexers globally ahead of time**
|
|
95
|
+
|
|
96
|
+
Same indexing steps as Option A; this only avoids `npx` download on the first run:
|
|
94
97
|
|
|
95
98
|
```bash
|
|
96
99
|
# TypeScript/JavaScript indexer (also handles plain JS via --infer-tsconfig)
|
|
@@ -127,7 +130,7 @@ scip-cli <command> [arguments]
|
|
|
127
130
|
- `symbols <file>` - List all symbols in a file (`--path`; bare filename OK)
|
|
128
131
|
- `rdeps <file>` - Find files that depend on a file (`--path`)
|
|
129
132
|
- `members <symbol>` - List members of a class/interface (`--path`)
|
|
130
|
-
- `reindex` - Force re-indexing of the current project
|
|
133
|
+
- `reindex` - Force re-indexing of the current project (`--path` to limit scope; repeatable)
|
|
131
134
|
- `skill [path]` - Install or dump the SKILL.md
|
|
132
135
|
|
|
133
136
|
### Examples
|
|
@@ -160,7 +163,7 @@ scip-cli skill ~/.claude/skills/scip-cli/SKILL.md
|
|
|
160
163
|
|
|
161
164
|
### Pipelines
|
|
162
165
|
|
|
163
|
-
Stdout is one record per line; stderr carries warnings. Pipe-friendly flags: `refs --paths-only`, `search --names-only` / `--paths-only`, `members --names-only`. `rdeps` already prints bare file paths.
|
|
166
|
+
Stdout is one record per line; stderr carries warnings and ambiguity notices. Kinds are lowercase (`function`, `class`, `method`, `property`, `variable`). Pipe-friendly flags: `refs --paths-only`, `search --names-only` / `--paths-only`, `members --names-only`. `rdeps` already prints bare file paths.
|
|
164
167
|
|
|
165
168
|
```bash
|
|
166
169
|
# What do importers of this file export?
|
|
@@ -180,10 +183,10 @@ scip-cli members Widget --names-only | xargs -I{} scip-cli def Widget.{}
|
|
|
180
183
|
|
|
181
184
|
1. On first query, automatically detects project language from `package.json` (TS/JS) or `pyproject.toml`/`setup.py` (Python)
|
|
182
185
|
2. For TypeScript monorepos, walks the repository for `tsconfig*.json` project roots (nested ancestors deduped; root included only when its `include` is broad)
|
|
183
|
-
3.
|
|
184
|
-
4. Converts
|
|
185
|
-
5. Caches the
|
|
186
|
-
6. Subsequent queries are
|
|
186
|
+
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial), or `scip-python` for Python
|
|
187
|
+
4. Converts each SCIP output to SQLite with `scip expt-convert`, then merges partial databases when needed
|
|
188
|
+
5. Caches the result in `~/.cache/scip-cli/projects/<dirname>-<hash>/index.db` (e.g. `my-monorepo-1a3f7a`)
|
|
189
|
+
6. Subsequent queries are SQLite lookups against that cache (not re-indexing)
|
|
187
190
|
|
|
188
191
|
## Configuration
|
|
189
192
|
|
|
@@ -192,16 +195,27 @@ Optional `.scip-cli.json` in the project root:
|
|
|
192
195
|
```json
|
|
193
196
|
{
|
|
194
197
|
"maxHeapMb": 8192,
|
|
195
|
-
"indexRoots": ["packages/
|
|
198
|
+
"indexRoots": ["packages/core", "apps/worker"],
|
|
196
199
|
"onlyIndexRoots": false
|
|
197
200
|
}
|
|
198
201
|
```
|
|
199
202
|
|
|
200
203
|
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage.
|
|
201
|
-
- `indexRoots` — extra TypeScript project directories to index
|
|
202
|
-
- `onlyIndexRoots` — skip auto-discovery and index only `indexRoots` (
|
|
204
|
+
- `indexRoots` — extra TypeScript project directories to include on **first index**, merged with auto-discovered projects.
|
|
205
|
+
- `onlyIndexRoots` — skip auto-discovery and index **only** `indexRoots` (smaller initial index when you only care about part of a monorepo).
|
|
206
|
+
|
|
207
|
+
`SCIP_CLI_INDEX_WORKERS` controls parallel `scip-typescript` runs during first index (default: up to 8). Merge into one database is always serial.
|
|
208
|
+
|
|
209
|
+
Scoped indexing without editing `.scip-cli.json`:
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
scip-cli reindex --path entrypoints/server
|
|
213
|
+
scip-cli reindex --path packages/api --path packages/worker
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
203
217
|
|
|
204
|
-
|
|
218
|
+
Run `scip-cli reindex` after changing scope, `.scip-cli.json` index settings, or when you want a fresh index.
|
|
205
219
|
|
|
206
220
|
This is separate from `.scipquery.json`, which belongs to [scip-query](https://github.com/PlunderStruck/scip-query) and configures its analyzers, watch mode, and diff-gate — not read by scip-cli.
|
|
207
221
|
|
|
@@ -55,18 +55,21 @@ python -m scip_cli --help
|
|
|
55
55
|
|
|
56
56
|
### 2. Install prerequisites (optional)
|
|
57
57
|
|
|
58
|
-
scip-cli
|
|
58
|
+
On **first index**, scip-cli runs language indexers and builds a SQLite cache. You can let it fetch tools on demand, or install them globally ahead of time so the first run does not download via `npx`:
|
|
59
59
|
|
|
60
60
|
**Option A: Zero extra setup (recommended)**
|
|
61
61
|
|
|
62
62
|
Install `scip-cli` and run it. On first index, scip-cli will:
|
|
63
63
|
|
|
64
|
-
- Download `scip-typescript` / `scip-python` via `npx` when
|
|
64
|
+
- Download `scip-typescript` / `scip-python` via `npx` when not already on PATH
|
|
65
65
|
- Download the `scip` converter binary from [GitHub releases](https://github.com/scip-code/scip/releases) into `~/.cache/scip-cli/bin/` when not already on PATH
|
|
66
|
+
- Walk the repo for `tsconfig*.json` project roots (TypeScript monorepos), run `scip-typescript` per project (parallel by default), convert each partial index, then merge into one `index.db`
|
|
66
67
|
|
|
67
|
-
No `.scip-cli.json` required
|
|
68
|
+
No `.scip-cli.json` required for discovery. Subsequent queries read the cached database only.
|
|
68
69
|
|
|
69
|
-
**Option B: Install globally
|
|
70
|
+
**Option B: Install indexers globally ahead of time**
|
|
71
|
+
|
|
72
|
+
Same indexing steps as Option A; this only avoids `npx` download on the first run:
|
|
70
73
|
|
|
71
74
|
```bash
|
|
72
75
|
# TypeScript/JavaScript indexer (also handles plain JS via --infer-tsconfig)
|
|
@@ -103,7 +106,7 @@ scip-cli <command> [arguments]
|
|
|
103
106
|
- `symbols <file>` - List all symbols in a file (`--path`; bare filename OK)
|
|
104
107
|
- `rdeps <file>` - Find files that depend on a file (`--path`)
|
|
105
108
|
- `members <symbol>` - List members of a class/interface (`--path`)
|
|
106
|
-
- `reindex` - Force re-indexing of the current project
|
|
109
|
+
- `reindex` - Force re-indexing of the current project (`--path` to limit scope; repeatable)
|
|
107
110
|
- `skill [path]` - Install or dump the SKILL.md
|
|
108
111
|
|
|
109
112
|
### Examples
|
|
@@ -136,7 +139,7 @@ scip-cli skill ~/.claude/skills/scip-cli/SKILL.md
|
|
|
136
139
|
|
|
137
140
|
### Pipelines
|
|
138
141
|
|
|
139
|
-
Stdout is one record per line; stderr carries warnings. Pipe-friendly flags: `refs --paths-only`, `search --names-only` / `--paths-only`, `members --names-only`. `rdeps` already prints bare file paths.
|
|
142
|
+
Stdout is one record per line; stderr carries warnings and ambiguity notices. Kinds are lowercase (`function`, `class`, `method`, `property`, `variable`). Pipe-friendly flags: `refs --paths-only`, `search --names-only` / `--paths-only`, `members --names-only`. `rdeps` already prints bare file paths.
|
|
140
143
|
|
|
141
144
|
```bash
|
|
142
145
|
# What do importers of this file export?
|
|
@@ -156,10 +159,10 @@ scip-cli members Widget --names-only | xargs -I{} scip-cli def Widget.{}
|
|
|
156
159
|
|
|
157
160
|
1. On first query, automatically detects project language from `package.json` (TS/JS) or `pyproject.toml`/`setup.py` (Python)
|
|
158
161
|
2. For TypeScript monorepos, walks the repository for `tsconfig*.json` project roots (nested ancestors deduped; root included only when its `include` is broad)
|
|
159
|
-
3.
|
|
160
|
-
4. Converts
|
|
161
|
-
5. Caches the
|
|
162
|
-
6. Subsequent queries are
|
|
162
|
+
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial), or `scip-python` for Python
|
|
163
|
+
4. Converts each SCIP output to SQLite with `scip expt-convert`, then merges partial databases when needed
|
|
164
|
+
5. Caches the result in `~/.cache/scip-cli/projects/<dirname>-<hash>/index.db` (e.g. `my-monorepo-1a3f7a`)
|
|
165
|
+
6. Subsequent queries are SQLite lookups against that cache (not re-indexing)
|
|
163
166
|
|
|
164
167
|
## Configuration
|
|
165
168
|
|
|
@@ -168,16 +171,27 @@ Optional `.scip-cli.json` in the project root:
|
|
|
168
171
|
```json
|
|
169
172
|
{
|
|
170
173
|
"maxHeapMb": 8192,
|
|
171
|
-
"indexRoots": ["packages/
|
|
174
|
+
"indexRoots": ["packages/core", "apps/worker"],
|
|
172
175
|
"onlyIndexRoots": false
|
|
173
176
|
}
|
|
174
177
|
```
|
|
175
178
|
|
|
176
179
|
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage.
|
|
177
|
-
- `indexRoots` — extra TypeScript project directories to index
|
|
178
|
-
- `onlyIndexRoots` — skip auto-discovery and index only `indexRoots` (
|
|
180
|
+
- `indexRoots` — extra TypeScript project directories to include on **first index**, merged with auto-discovered projects.
|
|
181
|
+
- `onlyIndexRoots` — skip auto-discovery and index **only** `indexRoots` (smaller initial index when you only care about part of a monorepo).
|
|
182
|
+
|
|
183
|
+
`SCIP_CLI_INDEX_WORKERS` controls parallel `scip-typescript` runs during first index (default: up to 8). Merge into one database is always serial.
|
|
184
|
+
|
|
185
|
+
Scoped indexing without editing `.scip-cli.json`:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
scip-cli reindex --path entrypoints/server
|
|
189
|
+
scip-cli reindex --path packages/api --path packages/worker
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
179
193
|
|
|
180
|
-
|
|
194
|
+
Run `scip-cli reindex` after changing scope, `.scip-cli.json` index settings, or when you want a fresh index.
|
|
181
195
|
|
|
182
196
|
This is separate from `.scipquery.json`, which belongs to [scip-query](https://github.com/PlunderStruck/scip-query) and configures its analyzers, watch mode, and diff-gate — not read by scip-cli.
|
|
183
197
|
|
|
@@ -22,7 +22,7 @@ All commands are sub-commands of `scip-cli`. Run from the project root.
|
|
|
22
22
|
|
|
23
23
|
- **Bare names** resolve functions, types (aliases + interfaces), and classes. Use dotted qualifiers to disambiguate members: `def Widget.run`, `refs Foo.setBar`, `search MyClass.myMethod`, `members pkg.MyClass`. Consts/variables need `def --kind variable X` or `search --kind variable X`. Class methods need `members ClassName`, not bare `def methodName`.
|
|
24
24
|
- **Ambiguous types** (e.g. `Opts` in multiple hooks) — `def` returns all matches; `refs` returns refs for all matching symbols. Use `--limit N` to cap results, or use `search` with a more specific pattern to disambiguate.
|
|
25
|
-
- **First run** in a project may auto-index (one-time wait; large monorepos with many `tsconfig.json` files take longer). JS-only projects (no `tsconfig.json`) are supported automatically.
|
|
25
|
+
- **First run** in a project may auto-index (one-time wait; large monorepos with many `tsconfig.json` files take longer). Projects index in parallel by default (`SCIP_CLI_INDEX_WORKERS`; merge is serial). JS-only projects (no `tsconfig.json`) are supported automatically.
|
|
26
26
|
- **Monorepos** are indexed by walking for `tsconfig*.json` under the repo (skips `node_modules`, `.git`, etc.). Nested parent/child projects are deduped. Add extra roots or limit indexing with `.scip-cli.json` (see README). Use `--path packages/api` (or any file/dir) to scope queries.
|
|
27
27
|
- **Prerequisites**: Node.js (for `npx` indexers). The `scip` converter auto-downloads on first use if missing; `scip-typescript` / `scip-python` download via `npx`. Optional `.scip-cli.json` for extra index roots or heap tuning. `brew install scip` installs an unrelated optimization solver — scip-cli ignores it and downloads the real binary.
|
|
28
28
|
|
|
@@ -48,7 +48,7 @@ refs [--limit N] [--path PATH] [--paths-only] <symbol>
|
|
|
48
48
|
|
|
49
49
|
Returns `file:line` for each reference. Reads source files to find exact line numbers.
|
|
50
50
|
|
|
51
|
-
Default `--limit` is 10. When multiple symbols match, refs are grouped by symbol with `# <
|
|
51
|
+
Default `--limit` is 10. When multiple symbols match, refs are grouped by symbol with `# <leaf-name>` headers on **stderr** (stdout stays pipe-clean). Use `--paths-only` for unique file paths (pipe-friendly).
|
|
52
52
|
|
|
53
53
|
### Pipelines
|
|
54
54
|
|
|
@@ -75,7 +75,7 @@ Each `xargs` invocation reopens the index (fast on cache hit). Use `--limit` on
|
|
|
75
75
|
search [--kind <kind>] [--limit N] [--path PATH] [--names-only] [--paths-only] <pattern>
|
|
76
76
|
```
|
|
77
77
|
|
|
78
|
-
Returns `file:line
|
|
78
|
+
Returns `file:line kind symbolName` (kinds are lowercase: `function`, `class`, etc.). Filters noisy symbols (file-level, parameters, type literals).
|
|
79
79
|
|
|
80
80
|
Default `--limit` is 10.
|
|
81
81
|
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""scip-cli: Fast code intelligence via SCIP indexes."""
|
|
2
|
-
__version__ = "1.
|
|
2
|
+
__version__ = "1.3.0"
|
|
@@ -6,7 +6,7 @@ import sqlite3
|
|
|
6
6
|
import sys
|
|
7
7
|
|
|
8
8
|
from . import __version__
|
|
9
|
-
from .cli_args import add_names_only_argument, add_path_argument, add_paths_only_argument
|
|
9
|
+
from .cli_args import add_names_only_argument, add_path_argument, add_paths_only_argument
|
|
10
10
|
from .symbols import SymbolKind
|
|
11
11
|
from .commands import refs, def_cmd, search, symbols, rdeps, members, skill, reindex
|
|
12
12
|
|
|
@@ -87,7 +87,13 @@ def main():
|
|
|
87
87
|
skill_parser.add_argument("path", nargs="?", help="Optional file path to write to (creates dirs)")
|
|
88
88
|
|
|
89
89
|
# reindex
|
|
90
|
-
subparsers.add_parser("reindex", help="Force re-indexing of the current project")
|
|
90
|
+
reindex_parser = subparsers.add_parser("reindex", help="Force re-indexing of the current project")
|
|
91
|
+
reindex_parser.add_argument(
|
|
92
|
+
"--path",
|
|
93
|
+
action="append",
|
|
94
|
+
metavar="PATH",
|
|
95
|
+
help="Index only tsconfig projects under PATH (repeatable; persisted until full reindex)",
|
|
96
|
+
)
|
|
91
97
|
|
|
92
98
|
args = parser.parse_args()
|
|
93
99
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Index cache path resolution."""
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from .project import find_project_root
|
|
6
|
+
from .scope import project_root_hash
|
|
7
|
+
|
|
8
|
+
INDEX_DB = "index.db"
|
|
9
|
+
INDEX_DB_NEXT = "index.db.next"
|
|
10
|
+
CACHE_SLUG_MAX_LEN = 48
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def index_db_path(cache_dir: Path, *, replace: bool = False) -> Path:
|
|
14
|
+
return Path(cache_dir) / (INDEX_DB_NEXT if replace else INDEX_DB)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _unlink_sqlite_sidecars(db_path: Path) -> None:
|
|
18
|
+
for suffix in ("-wal", "-shm"):
|
|
19
|
+
sidecar = Path(f"{db_path}{suffix}")
|
|
20
|
+
if sidecar.is_file():
|
|
21
|
+
sidecar.unlink()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def cleanup_in_progress_index(cache_dir: Path) -> None:
|
|
25
|
+
"""Remove a failed or abandoned index.db.next build."""
|
|
26
|
+
next_db = index_db_path(cache_dir, replace=True)
|
|
27
|
+
if next_db.is_file():
|
|
28
|
+
next_db.unlink()
|
|
29
|
+
_unlink_sqlite_sidecars(next_db)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def promote_next_index(cache_dir: Path) -> None:
|
|
33
|
+
"""Atomically swap index.db.next over the live index.db."""
|
|
34
|
+
cache_dir = Path(cache_dir)
|
|
35
|
+
next_db = index_db_path(cache_dir, replace=True)
|
|
36
|
+
live_db = index_db_path(cache_dir, replace=False)
|
|
37
|
+
if not next_db.is_file():
|
|
38
|
+
raise RuntimeError("index.db.next is missing")
|
|
39
|
+
|
|
40
|
+
_unlink_sqlite_sidecars(live_db)
|
|
41
|
+
if live_db.is_file():
|
|
42
|
+
live_db.unlink()
|
|
43
|
+
next_db.rename(live_db)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def project_cache_slug(project_root: Path) -> str:
|
|
47
|
+
"""Human-readable cache directory name for a project root."""
|
|
48
|
+
root = Path(project_root).resolve()
|
|
49
|
+
parts = root.parts
|
|
50
|
+
slug_base = root.name or (parts[-1] if parts else "project")
|
|
51
|
+
slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", slug_base).strip("-") or "project"
|
|
52
|
+
if len(slug) > CACHE_SLUG_MAX_LEN:
|
|
53
|
+
slug = slug[:CACHE_SLUG_MAX_LEN].rstrip("-")
|
|
54
|
+
digest = project_root_hash(root)[:6]
|
|
55
|
+
return f"{slug}-{digest}"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def get_cache_dir(project_root):
|
|
59
|
+
"""Get the cache directory for a project (one dir per repo root)."""
|
|
60
|
+
root = Path(project_root).resolve()
|
|
61
|
+
return Path.home() / ".cache" / "scip-cli" / "projects" / project_cache_slug(root)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def find_db(project_root=None):
|
|
65
|
+
"""Find the index.db for the given project (or cwd)."""
|
|
66
|
+
root = project_root or find_project_root()
|
|
67
|
+
if not root:
|
|
68
|
+
return None
|
|
69
|
+
cache = index_db_path(get_cache_dir(root), replace=False)
|
|
70
|
+
return cache if cache.is_file() else None
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Force re-indexing of the current project."""
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from ..cache import (
|
|
5
|
+
cleanup_in_progress_index,
|
|
6
|
+
get_cache_dir,
|
|
7
|
+
index_db_path,
|
|
8
|
+
promote_next_index,
|
|
9
|
+
)
|
|
10
|
+
from ..indexing import _index_project
|
|
11
|
+
from ..paths import normalize_path_scope
|
|
12
|
+
from ..project import detect_language, find_project_root
|
|
13
|
+
from ..scope import save_index_scope
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main(args):
|
|
17
|
+
root = find_project_root()
|
|
18
|
+
if not root:
|
|
19
|
+
print("Error: Could not find project root", file=sys.stderr)
|
|
20
|
+
sys.exit(1)
|
|
21
|
+
|
|
22
|
+
path_args = getattr(args, "path", None) or []
|
|
23
|
+
if path_args:
|
|
24
|
+
scope_paths = [normalize_path_scope(path, root) for path in path_args]
|
|
25
|
+
save_index_scope(root, scope_paths)
|
|
26
|
+
print(f"Index scope: {', '.join(scope_paths)}", file=sys.stderr)
|
|
27
|
+
else:
|
|
28
|
+
save_index_scope(root, None)
|
|
29
|
+
|
|
30
|
+
cache_dir = get_cache_dir(root)
|
|
31
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
cleanup_in_progress_index(cache_dir)
|
|
33
|
+
|
|
34
|
+
lang = detect_language(root)
|
|
35
|
+
if lang is None:
|
|
36
|
+
print(f"Error: No supported project markers found in {root}", file=sys.stderr)
|
|
37
|
+
sys.exit(1)
|
|
38
|
+
|
|
39
|
+
print(f"Re-indexing {root} ({lang})...", file=sys.stderr)
|
|
40
|
+
try:
|
|
41
|
+
_index_project(root, lang, cache_dir, replace=True)
|
|
42
|
+
except RuntimeError as e:
|
|
43
|
+
cleanup_in_progress_index(cache_dir)
|
|
44
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
45
|
+
sys.exit(1)
|
|
46
|
+
|
|
47
|
+
next_db = index_db_path(cache_dir, replace=True)
|
|
48
|
+
if not next_db.is_file():
|
|
49
|
+
cleanup_in_progress_index(cache_dir)
|
|
50
|
+
print("Error: No index.db found after indexing", file=sys.stderr)
|
|
51
|
+
sys.exit(1)
|
|
52
|
+
|
|
53
|
+
promote_next_index(cache_dir)
|
|
54
|
+
print(f"Updated {index_db_path(cache_dir, replace=False)}", file=sys.stderr)
|
|
55
|
+
print("Re-indexing complete", file=sys.stderr)
|
|
@@ -43,7 +43,7 @@ def is_noisy_symbol(symbol_str):
|
|
|
43
43
|
return True
|
|
44
44
|
if symbol_str.endswith("/__init__:"):
|
|
45
45
|
return True
|
|
46
|
-
if "typeLiteral" in symbol_str:
|
|
46
|
+
if "typeLiteral" in symbol_str and infer_kind(symbol_str) != SymbolKind.PROPERTY:
|
|
47
47
|
return True
|
|
48
48
|
if ").(" in symbol_str:
|
|
49
49
|
return True
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
"""Shared constants for scip-cli."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
|
|
3
5
|
INDEX_TIMEOUT = 300
|
|
4
6
|
DEFAULT_MAX_HEAP_MB = 8192
|
|
5
7
|
DEFAULT_MAX_DEF_LINES = 80
|
|
6
8
|
DEFAULT_MAX_DEF_CHARS = 32_000
|
|
7
9
|
SCIP_INSTALL_URL = "https://github.com/scip-code/scip/releases"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def default_index_workers() -> int:
|
|
13
|
+
"""Default parallel TypeScript project indexers (merge stays single-threaded)."""
|
|
14
|
+
return min(8, os.cpu_count() or 4)
|
|
@@ -8,12 +8,14 @@ import sqlite3
|
|
|
8
8
|
import subprocess
|
|
9
9
|
import sys
|
|
10
10
|
import tempfile
|
|
11
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
|
|
13
|
-
from .cache import find_db, get_cache_dir
|
|
14
|
+
from .cache import find_db, get_cache_dir, index_db_path
|
|
14
15
|
from .config import CONFIG_FILENAME, load_project_config, resolve_index_roots
|
|
15
|
-
from .constants import DEFAULT_MAX_HEAP_MB, INDEX_TIMEOUT, SCIP_INSTALL_URL
|
|
16
|
+
from .constants import DEFAULT_MAX_HEAP_MB, INDEX_TIMEOUT, SCIP_INSTALL_URL, default_index_workers
|
|
16
17
|
from .discover import discover_typescript_projects
|
|
18
|
+
from .scope import load_index_scope, projects_matching_scope
|
|
17
19
|
from .merge import merge_sqlite_indexes
|
|
18
20
|
from .project import detect_language, find_project_root
|
|
19
21
|
from .scip_tool import ensure_scip_binary
|
|
@@ -73,6 +75,16 @@ def indexer_env(project_root=None):
|
|
|
73
75
|
def typescript_projects(root: Path) -> list[Path]:
|
|
74
76
|
"""Resolve the TypeScript project list for a repository."""
|
|
75
77
|
settings = load_project_config(root)
|
|
78
|
+
scope = load_index_scope(root)
|
|
79
|
+
|
|
80
|
+
if scope is not None:
|
|
81
|
+
discovered = list(discover_typescript_projects(root))
|
|
82
|
+
filtered = projects_matching_scope(discovered, scope.paths)
|
|
83
|
+
if not filtered:
|
|
84
|
+
joined = ", ".join(scope.paths)
|
|
85
|
+
raise RuntimeError(f"No TypeScript projects found under index scope: {joined}")
|
|
86
|
+
return sorted(filtered, key=str)
|
|
87
|
+
|
|
76
88
|
configured = resolve_index_roots(root, settings) if settings.index_roots else []
|
|
77
89
|
|
|
78
90
|
if settings.only_index_roots:
|
|
@@ -135,11 +147,10 @@ def _warn_old_scip(binary):
|
|
|
135
147
|
)
|
|
136
148
|
|
|
137
149
|
|
|
138
|
-
def _convert_scip_to_db(scip_path,
|
|
139
|
-
"""Convert a SCIP protobuf file to SQLite
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
db_path = cache_dir / "index.db"
|
|
150
|
+
def _convert_scip_to_db(scip_path, db_path):
|
|
151
|
+
"""Convert a SCIP protobuf file to a SQLite index at db_path."""
|
|
152
|
+
db_path = Path(db_path)
|
|
153
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
143
154
|
if db_path.exists():
|
|
144
155
|
db_path.unlink()
|
|
145
156
|
|
|
@@ -147,8 +158,8 @@ def _convert_scip_to_db(scip_path, cache_dir):
|
|
|
147
158
|
_warn_old_scip(scip_binary)
|
|
148
159
|
|
|
149
160
|
result = _run_subprocess(
|
|
150
|
-
[scip_binary, "expt-convert", str(scip_path), "--output",
|
|
151
|
-
cwd=str(
|
|
161
|
+
[scip_binary, "expt-convert", str(scip_path), "--output", db_path.name],
|
|
162
|
+
cwd=str(db_path.parent),
|
|
152
163
|
)
|
|
153
164
|
if result.returncode != 0:
|
|
154
165
|
print(result.stderr, file=sys.stderr)
|
|
@@ -166,65 +177,132 @@ def _typescript_index_args(root, output_scip, projects):
|
|
|
166
177
|
return args
|
|
167
178
|
|
|
168
179
|
|
|
169
|
-
def
|
|
170
|
-
"""
|
|
180
|
+
def _index_workers():
|
|
181
|
+
"""Parallel workers for per-project scip-typescript runs (merge stays serial)."""
|
|
182
|
+
env_val = os.environ.get("SCIP_CLI_INDEX_WORKERS")
|
|
183
|
+
if env_val is not None:
|
|
184
|
+
try:
|
|
185
|
+
return max(1, int(env_val))
|
|
186
|
+
except ValueError:
|
|
187
|
+
raise RuntimeError(
|
|
188
|
+
f"Invalid SCIP_CLI_INDEX_WORKERS: expected an integer, got {env_val!r}"
|
|
189
|
+
) from None
|
|
190
|
+
return default_index_workers()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _index_one_ts_project(root, project, work_dir, env):
|
|
194
|
+
"""Index one TypeScript project into work_dir/index.db."""
|
|
195
|
+
root = Path(root)
|
|
196
|
+
work_dir = Path(work_dir)
|
|
197
|
+
work_dir.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
label = "." if project == Path(".") else str(project)
|
|
199
|
+
part_scip = work_dir / "index.scip"
|
|
200
|
+
index_args = _typescript_index_args(root, part_scip, [project])
|
|
201
|
+
result = run_with_fallback(
|
|
202
|
+
"scip-typescript",
|
|
203
|
+
"@sourcegraph/scip-typescript",
|
|
204
|
+
str(root),
|
|
205
|
+
index_args,
|
|
206
|
+
env=env,
|
|
207
|
+
)
|
|
208
|
+
if result.returncode != 0:
|
|
209
|
+
return label, None, result.stderr.strip() or "indexing failed"
|
|
210
|
+
_convert_scip_to_db(part_scip, work_dir / "index.db")
|
|
211
|
+
return label, work_dir / "index.db", None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _index_typescript(root, cache_dir, projects, env, *, replace=False):
|
|
215
|
+
"""Index one or more TypeScript projects and write the merged index.db."""
|
|
171
216
|
root = Path(root)
|
|
172
217
|
cache_dir = Path(cache_dir)
|
|
218
|
+
output_db = index_db_path(cache_dir, replace=replace)
|
|
219
|
+
workers = _index_workers()
|
|
220
|
+
use_parallel = len(projects) > 1 and workers > 1
|
|
173
221
|
|
|
174
222
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
175
223
|
tmpdir_path = Path(tmpdir)
|
|
176
224
|
part_dbs: list[Path] = []
|
|
177
225
|
skipped = 0
|
|
226
|
+
total = len(projects)
|
|
178
227
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
f"Indexing TypeScript project {index}/{len(projects)}: {label}",
|
|
184
|
-
file=sys.stderr,
|
|
185
|
-
)
|
|
186
|
-
|
|
187
|
-
part_scip = tmpdir_path / f"part-{index}.scip"
|
|
188
|
-
index_args = _typescript_index_args(root, part_scip, [project])
|
|
189
|
-
result = run_with_fallback(
|
|
190
|
-
"scip-typescript",
|
|
191
|
-
"@sourcegraph/scip-typescript",
|
|
192
|
-
str(root),
|
|
193
|
-
index_args,
|
|
194
|
-
env=env,
|
|
228
|
+
if use_parallel:
|
|
229
|
+
print(
|
|
230
|
+
f"Indexing {total} TypeScript projects ({workers} workers; merge is serial)...",
|
|
231
|
+
file=sys.stderr,
|
|
195
232
|
)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
233
|
+
completed = 0
|
|
234
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
235
|
+
futures = {
|
|
236
|
+
pool.submit(
|
|
237
|
+
_index_one_ts_project,
|
|
238
|
+
root,
|
|
239
|
+
project,
|
|
240
|
+
tmpdir_path / f"part-{index}",
|
|
241
|
+
env,
|
|
242
|
+
): index
|
|
243
|
+
for index, project in enumerate(projects, start=1)
|
|
244
|
+
}
|
|
245
|
+
for future in as_completed(futures):
|
|
246
|
+
index = futures[future]
|
|
247
|
+
label, db_path, error = future.result()
|
|
248
|
+
completed += 1
|
|
249
|
+
if db_path is None:
|
|
250
|
+
skipped += 1
|
|
251
|
+
print(
|
|
252
|
+
f"Warning: skipped {label}: {error}",
|
|
253
|
+
file=sys.stderr,
|
|
254
|
+
)
|
|
255
|
+
else:
|
|
256
|
+
part_dbs.append((index, db_path))
|
|
257
|
+
print(
|
|
258
|
+
f"Indexed {completed}/{total}: {label}",
|
|
259
|
+
file=sys.stderr,
|
|
260
|
+
)
|
|
261
|
+
part_dbs = [db for _, db in sorted(part_dbs, key=lambda item: item[0])]
|
|
262
|
+
else:
|
|
263
|
+
for index, project in enumerate(projects, start=1):
|
|
264
|
+
label = "." if project == Path(".") else str(project)
|
|
265
|
+
if total > 1:
|
|
266
|
+
print(
|
|
267
|
+
f"Indexing TypeScript project {index}/{total}: {label}",
|
|
268
|
+
file=sys.stderr,
|
|
269
|
+
)
|
|
270
|
+
label, db_path, error = _index_one_ts_project(
|
|
271
|
+
root,
|
|
272
|
+
project,
|
|
273
|
+
tmpdir_path / f"part-{index}",
|
|
274
|
+
env,
|
|
201
275
|
)
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
276
|
+
if db_path is None:
|
|
277
|
+
skipped += 1
|
|
278
|
+
print(f"Warning: skipped {label}: {error}", file=sys.stderr)
|
|
279
|
+
continue
|
|
280
|
+
part_dbs.append(db_path)
|
|
207
281
|
|
|
208
282
|
if not part_dbs:
|
|
209
283
|
raise RuntimeError("Failed to index project")
|
|
210
284
|
|
|
211
285
|
indexed = len(part_dbs)
|
|
212
|
-
total
|
|
213
|
-
if total > 1:
|
|
286
|
+
if total > 1 and not use_parallel:
|
|
214
287
|
print(
|
|
215
288
|
f"Indexed {indexed}/{total} TypeScript projects"
|
|
216
289
|
+ (f" ({skipped} skipped)" if skipped else ""),
|
|
217
290
|
file=sys.stderr,
|
|
218
291
|
)
|
|
292
|
+
elif total > 1 and use_parallel:
|
|
293
|
+
print(
|
|
294
|
+
f"Finished indexing {indexed}/{total} TypeScript projects"
|
|
295
|
+
+ (f" ({skipped} skipped)" if skipped else ""),
|
|
296
|
+
file=sys.stderr,
|
|
297
|
+
)
|
|
219
298
|
|
|
220
299
|
if len(part_dbs) == 1:
|
|
221
|
-
shutil.copy2(part_dbs[0],
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
merge_sqlite_indexes(part_dbs, cache_dir / "index.db")
|
|
300
|
+
shutil.copy2(part_dbs[0], output_db)
|
|
301
|
+
else:
|
|
302
|
+
merge_sqlite_indexes(part_dbs, output_db)
|
|
225
303
|
|
|
226
304
|
|
|
227
|
-
def _index_project(root, lang, cache_dir):
|
|
305
|
+
def _index_project(root, lang, cache_dir, *, replace=False):
|
|
228
306
|
"""Run the language-specific indexer and convert to DB."""
|
|
229
307
|
cache_dir = Path(cache_dir)
|
|
230
308
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -238,7 +316,7 @@ def _index_project(root, lang, cache_dir):
|
|
|
238
316
|
f"Indexing {len(projects)} TypeScript projects",
|
|
239
317
|
file=sys.stderr,
|
|
240
318
|
)
|
|
241
|
-
_index_typescript(root, cache_dir, projects, env)
|
|
319
|
+
_index_typescript(root, cache_dir, projects, env, replace=replace)
|
|
242
320
|
return
|
|
243
321
|
|
|
244
322
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
@@ -258,7 +336,7 @@ def _index_project(root, lang, cache_dir):
|
|
|
258
336
|
print(result.stderr, file=sys.stderr)
|
|
259
337
|
raise RuntimeError("Failed to index project")
|
|
260
338
|
|
|
261
|
-
_convert_scip_to_db(index_scip, cache_dir)
|
|
339
|
+
_convert_scip_to_db(index_scip, index_db_path(cache_dir, replace=replace))
|
|
262
340
|
|
|
263
341
|
|
|
264
342
|
def get_db(project_root=None):
|