scip-cli 2.4.0__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.cursor/rules/agent.mdc +20 -0
- scip_cli-2.5.0/.github/workflows/ci.yml +22 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.pre-commit-config.yaml +1 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/PKG-INFO +40 -22
- {scip_cli-2.4.0 → scip_cli-2.5.0}/README.md +39 -21
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/SKILL.md +4 -4
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/__init__.py +1 -1
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/live.py +15 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/project.py +8 -1
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/reindex.py +1 -1
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/discover.py +54 -0
- scip_cli-2.5.0/scip_cli/indexing/__init__.py +29 -0
- scip_cli-2.5.0/scip_cli/indexing/constants.py +13 -0
- scip_cli-2.5.0/scip_cli/indexing/convert.py +94 -0
- scip_cli-2.5.0/scip_cli/indexing/core.py +210 -0
- scip_cli-2.5.0/scip_cli/indexing/languages.py +85 -0
- scip_cli-2.5.0/scip_cli/indexing/orchestrate.py +158 -0
- scip_cli-2.5.0/scip_cli/indexing/postprocess.py +114 -0
- scip_cli-2.5.0/scip_cli/indexing/runners.py +116 -0
- scip_cli-2.5.0/scip_cli/indexing/typescript.py +165 -0
- scip_cli-2.5.0/scip_cli/merge.py +168 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/project.py +6 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/sql.py +11 -0
- scip_cli-2.5.0/scripts/bench_postprocess.py +236 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scripts/publish.sh +6 -0
- scip_cli-2.5.0/scripts/test.sh +29 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_discover.py +72 -0
- scip_cli-2.5.0/tests/test_index_batching.py +59 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_index_prune.py +40 -3
- scip_cli-2.5.0/tests/test_indexer_fallback.py +315 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_merge.py +36 -0
- scip_cli-2.5.0/tests/test_python_index_integration.py +88 -0
- scip_cli-2.4.0/scip_cli/indexing.py +0 -522
- scip_cli-2.4.0/scip_cli/merge.py +0 -108
- scip_cli-2.4.0/scripts/test.sh +0 -12
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.github/ISSUE_TEMPLATE/question.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.github/pull_request_template.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/.gitignore +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/CODE_OF_CONDUCT.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/CONTRIBUTING.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/LICENSE +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/SECURITY.md +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/pyproject.toml +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/__main__.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/__init__.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/common.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/file.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/graph.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/sections.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/symbol.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/analyze/targets.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/cache.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/cli_args.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/__init__.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/analyze.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/code.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/deps.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/members.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/rdeps.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/refs.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/search.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/skill.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/commands/symbols.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/config.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/debug.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/output.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/paths.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/queries.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/scip_tool.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/scope.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/session.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/source.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/symbols.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scip_cli/targets.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/scripts/build.sh +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/__init__.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/analyze_db.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/conftest.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/e2e_harness.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixture_catalog.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/package.json +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/app/handler.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/config.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/consumer.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/dead.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/domain/i18n/en.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/domain/i18n/index.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/domain/i18n/useLocale.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/domain/labels/index.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/events/streamTypes.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/gateway/index.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/helper.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/hooks/useHookA.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/hooks/useHookB.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/hooks/useItems.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/index.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/integrations/inferenceClient.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/loaders/useInference.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/pages/panelPage.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/rules/applyRule.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/types/a.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/types/b.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/ui/Button.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/ui/LazyPanel.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/ui/buttonConsumer.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/ui/menuModule.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/user.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/widget.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/src/widgets/OrphanWidget.ts +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/fixtures/sample-project/tsconfig.json +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/perf_util.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_analyze.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_analyze_graph.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_analyze_perf.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_cache.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_composability.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_config.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_e2e.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_e2e_analyze_patterns.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_e2e_perf.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_indexer_env.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_multi_symbol.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_pure_functions.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_qualified_symbols.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_reindex.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_scip_tool.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_scope.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_targets.py +0 -0
- {scip_cli-2.4.0 → scip_cli-2.5.0}/tests/test_typescript_projects.py +0 -0
|
@@ -30,6 +30,7 @@ Node.js + `npx` required for integration tests (`scip-typescript`). Optional: `S
|
|
|
30
30
|
|
|
31
31
|
| Task | Command |
|
|
32
32
|
|------|---------|
|
|
33
|
+
| **All checks** | `scripts/test.sh` (lint + format + types + tests) |
|
|
33
34
|
| Full suite | `pytest` |
|
|
34
35
|
| E2e loop | `pytest tests/test_e2e.py tests/test_e2e_analyze_patterns.py` |
|
|
35
36
|
| Lint / format | `ruff check .` / `ruff format .` |
|
|
@@ -37,6 +38,8 @@ Node.js + `npx` required for integration tests (`scip-typescript`). Optional: `S
|
|
|
37
38
|
|
|
38
39
|
Pre-commit: ruff + ruff-format + basedpyright + full pytest. E2e before commit on command changes.
|
|
39
40
|
|
|
41
|
+
**Use `scripts/test.sh`** before committing to catch all issues (lint, format, types, tests). This matches what pre-commit runs.
|
|
42
|
+
|
|
40
43
|
## Improving this repo (do this before review subagents)
|
|
41
44
|
|
|
42
45
|
**Dogfood on the repo itself** — faster and more honest than spawning reviewers:
|
|
@@ -187,3 +190,20 @@ If both layers help during development, **keep both until e2e passes**, then del
|
|
|
187
190
|
- User install + analyze guide → `README.md`
|
|
188
191
|
- Agent/user CLI reference → `scip_cli/SKILL.md` (must match `__main__.py` flags)
|
|
189
192
|
- After flag changes: update SKILL + README command list; run `scip-cli skill` to verify output
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
## Dogfood Notes
|
|
196
|
+
|
|
197
|
+
Future improvements from using the CLI on real projects.
|
|
198
|
+
|
|
199
|
+
### Workflow
|
|
200
|
+
|
|
201
|
+
Requires bare `scip-cli` mapped to local (`pip install -e ".[dev]"` from repo root).
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
scip-cli reindex
|
|
205
|
+
scip-cli analyze --limit 25
|
|
206
|
+
scip-cli analyze scip_cli/queries.py --limit 20 # drill into hubs
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Project-wide analyze skips test paths by default; `--include-tests` to include. See README **Finding easy wins with analyze**.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python-version }}
|
|
19
|
+
- name: Install dependencies
|
|
20
|
+
run: pip install -e ".[dev]"
|
|
21
|
+
- name: Run publish gate
|
|
22
|
+
run: ./scripts/test.sh
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scip-cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: Fast code intelligence via SCIP indexes
|
|
5
5
|
Project-URL: Homepage, https://github.com/flesler/scip-cli
|
|
6
6
|
Author: Ariel Flesler
|
|
@@ -24,7 +24,7 @@ Description-Content-Type: text/markdown
|
|
|
24
24
|
[](https://badge.fury.io/py/scip-cli)
|
|
25
25
|
[](https://opensource.org/licenses/MIT)
|
|
26
26
|
|
|
27
|
-
Token-efficient code intelligence for AI agents. Precise refs, definitions, and repo health analysis via SCIP indexes — TypeScript/JavaScript and
|
|
27
|
+
Token-efficient code intelligence for AI agents. Precise refs, definitions, and repo health analysis via SCIP indexes — TypeScript/JavaScript, Python, Go, and Rust.
|
|
28
28
|
|
|
29
29
|
## Why
|
|
30
30
|
|
|
@@ -90,7 +90,7 @@ On **first index**, scip-cli runs language indexers and builds a SQLite cache. Y
|
|
|
90
90
|
|
|
91
91
|
Install `scip-cli` and run it. On first index, scip-cli will:
|
|
92
92
|
|
|
93
|
-
- Download `scip-typescript` / `scip-python` via `npx` when not already on PATH
|
|
93
|
+
- Download `scip-typescript` / `scip-python` via `npx`, `scip-go` via `go install`, or `rust-analyzer` via `rustup` when not already on PATH
|
|
94
94
|
- Download the `scip` converter binary from [GitHub releases](https://github.com/scip-code/scip/releases) into `~/.cache/scip-cli/bin/` when not already on PATH
|
|
95
95
|
- Walk the repo for `tsconfig*.json` project roots (TypeScript monorepos), run `scip-typescript` per project (parallel by default), convert each partial index, then merge into one `index.db`
|
|
96
96
|
|
|
@@ -107,6 +107,12 @@ npm install -g @sourcegraph/scip-typescript
|
|
|
107
107
|
# Python indexer
|
|
108
108
|
npm install -g @sourcegraph/scip-python
|
|
109
109
|
|
|
110
|
+
# Go indexer
|
|
111
|
+
go install github.com/scip-code/scip-go/cmd/scip-go@latest
|
|
112
|
+
|
|
113
|
+
# Rust indexer (rust-analyzer with SCIP support)
|
|
114
|
+
rustup component add rust-analyzer
|
|
115
|
+
|
|
110
116
|
# SCIP CLI for index conversion (GitHub release — not on npm)
|
|
111
117
|
# https://github.com/scip-code/scip/releases (v0.8.1+ recommended)
|
|
112
118
|
```
|
|
@@ -200,9 +206,9 @@ scip-cli deps greet --paths-only | sort -u
|
|
|
200
206
|
|
|
201
207
|
## How It Works
|
|
202
208
|
|
|
203
|
-
1. On first query, automatically detects project language from `package.json` (TS/JS)
|
|
209
|
+
1. On first query, automatically detects project language from `package.json` (TS/JS), `pyproject.toml`/`setup.py` (Python), `go.mod` (Go), or `Cargo.toml` (Rust)
|
|
204
210
|
2. For TypeScript monorepos, walks the repository for `tsconfig*.json` project roots (nested ancestors deduped; root included only when its `include` is broad)
|
|
205
|
-
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial),
|
|
211
|
+
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial), `scip-python` for Python, `scip-go` for Go, or `rust-analyzer scip` for Rust
|
|
206
212
|
4. Converts each SCIP output to SQLite with `scip expt-convert`, then merges partial databases when needed
|
|
207
213
|
5. Caches the result in `~/.cache/scip-cli/projects/<dirname>-<hash>/index.db` (e.g. `my-monorepo-1a3f7a`)
|
|
208
214
|
6. Subsequent queries are SQLite lookups against that cache (not re-indexing)
|
|
@@ -219,11 +225,23 @@ Optional `.scip-cli.json` in the project root:
|
|
|
219
225
|
}
|
|
220
226
|
```
|
|
221
227
|
|
|
222
|
-
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage.
|
|
228
|
+
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage. Does not affect `scip-go`.
|
|
223
229
|
- `indexRoots` — extra TypeScript project directories to include on **first index**, merged with auto-discovered projects.
|
|
224
230
|
- `onlyIndexRoots` — skip auto-discovery and index **only** `indexRoots` (smaller initial index when you only care about part of a monorepo).
|
|
225
231
|
|
|
226
|
-
`SCIP_CLI_INDEX_WORKERS` controls parallel
|
|
232
|
+
`SCIP_CLI_INDEX_WORKERS` controls parallel indexer runs during first index (default: up to 8). Merge into one database is always serial.
|
|
233
|
+
|
|
234
|
+
Other environment variables:
|
|
235
|
+
|
|
236
|
+
| Variable | Purpose |
|
|
237
|
+
| ------------------------------ | --------------------------------------------------------------------------------------------- |
|
|
238
|
+
| `SCIP_CLI_MAX_HEAP_MB` | Node heap for `scip-typescript` / `scip-python` (overrides `maxHeapMb` in config) |
|
|
239
|
+
| `SCIP_CLI_TS_INDEX_BATCH_SIZE` | Split large TS repos into multiple `scip-typescript` runs (default: all tsconfigs in one run) |
|
|
240
|
+
| `SCIP_CLI_MERGE_BATCH_SIZE` | SQLite ATTACH batch size when merging part DBs (max 9) |
|
|
241
|
+
| `SCIP_CLI_MAX_DEF_LINES` | Max definition lines in `code` output |
|
|
242
|
+
| `SCIP_CLI_DEBUG` | Log SQL queries to stderr |
|
|
243
|
+
|
|
244
|
+
**Version policy:** only the `scip` converter (`expt-convert`) is pinned to the 0.8.x release line because it defines the SQLite schema. Language indexers (`scip-typescript`, `scip-python` via `npx`; `scip-go` via `go install @latest`) install at latest on first use. `rust-analyzer` installs via `rustup component add`.
|
|
227
245
|
|
|
228
246
|
Large monorepos (>10 tsconfig projects) log per-project progress to stderr during indexing; smaller repos stay quiet aside from the final `Indexed … (size)` line.
|
|
229
247
|
|
|
@@ -234,7 +252,7 @@ scip-cli reindex --path packages/server
|
|
|
234
252
|
scip-cli reindex --path packages/api --path packages/worker
|
|
235
253
|
```
|
|
236
254
|
|
|
237
|
-
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
255
|
+
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). **TypeScript only** — other languages reject `reindex --path`. The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
238
256
|
|
|
239
257
|
Run `scip-cli reindex` after changing scope, `.scip-cli.json` index settings, or when you want a fresh index.
|
|
240
258
|
|
|
@@ -251,24 +269,24 @@ scip-cli analyze --priority high --limit 25 # dead exports & cycles only
|
|
|
251
269
|
|
|
252
270
|
Sections are tagged `[high]`, `[medium]`, `[low]` and listed in that order.
|
|
253
271
|
|
|
254
|
-
| Tier
|
|
255
|
-
|
|
|
256
|
-
| **high**
|
|
257
|
-
| **medium** | Same-file only, change surface (file target)
|
|
258
|
-
| **low**
|
|
272
|
+
| Tier | Project sections | Action |
|
|
273
|
+
| ---------- | ---------------------------------------------------- | -------------------------------------------------------------------- |
|
|
274
|
+
| **high** | Cycles, unreferenced, dead exports, stale types | Nuke or fix cycles; delete unused; `_` prefix |
|
|
275
|
+
| **medium** | Same-file only, change surface (file target) | Module-private by usage |
|
|
276
|
+
| **low** | Test-only consumers, coupling, bottlenecks, hotspots | Noisy on Python (index omits many same-file calls); verify with `rg` |
|
|
259
277
|
|
|
260
278
|
Use `--priority high` for a quick gate; `--priority high,medium` adds context. File drill-down adds change surface and unused imports.
|
|
261
279
|
|
|
262
280
|
**What to look at first**
|
|
263
281
|
|
|
264
|
-
| Section
|
|
265
|
-
|
|
|
266
|
-
| **Cycles**
|
|
267
|
-
| **Unreferenced**
|
|
268
|
-
| **Dead exports**
|
|
269
|
-
| **Stale types**
|
|
270
|
-
| **Same-file only**
|
|
271
|
-
| **Test-only consumers** | Cross-file refs are all from tests — promote to e2e or accept as internal
|
|
282
|
+
| Section | Easy pickings |
|
|
283
|
+
| ----------------------- | ------------------------------------------------------------------------------------------------------ |
|
|
284
|
+
| **Cycles** | Import/mention cycles between production files — break the edge or extract shared code |
|
|
285
|
+
| **Unreferenced** | No usage in the index at all — delete |
|
|
286
|
+
| **Dead exports** | No external refs — delete or `_` prefix |
|
|
287
|
+
| **Stale types** | Classes/types with no external consumer in the index — verify in-file or type-only use before removing |
|
|
288
|
+
| **Same-file only** | Used only inside defining file — rename to `_` |
|
|
289
|
+
| **Test-only consumers** | Cross-file refs are all from tests — promote to e2e or accept as internal |
|
|
272
290
|
|
|
273
291
|
**Per-file or package drill-down** on hubs or suspects:
|
|
274
292
|
|
|
@@ -277,7 +295,7 @@ scip-cli analyze scip_cli/queries.py --limit 20 # file: scoped project + per-f
|
|
|
277
295
|
scip-cli analyze scip_cli --limit 15 # directory: scoped project + each file under it
|
|
278
296
|
```
|
|
279
297
|
|
|
280
|
-
`Dead exports in file` lists same-module symbols with no
|
|
298
|
+
`Dead exports in file` lists same-module symbols with no _external_ refs — module-private `_helpers` are filtered out. Remaining rows are worth a manual `rg` check.
|
|
281
299
|
|
|
282
300
|
**Defaults:** project-wide and directory analyze skip `tests/`, `*.test.*`, `*.spec.*`, `conftest.py`, and `__tests__/`. Pass `--include-tests` to include them. File-target analyze always includes that file.
|
|
283
301
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](https://badge.fury.io/py/scip-cli)
|
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
|
5
5
|
|
|
6
|
-
Token-efficient code intelligence for AI agents. Precise refs, definitions, and repo health analysis via SCIP indexes — TypeScript/JavaScript and
|
|
6
|
+
Token-efficient code intelligence for AI agents. Precise refs, definitions, and repo health analysis via SCIP indexes — TypeScript/JavaScript, Python, Go, and Rust.
|
|
7
7
|
|
|
8
8
|
## Why
|
|
9
9
|
|
|
@@ -69,7 +69,7 @@ On **first index**, scip-cli runs language indexers and builds a SQLite cache. Y
|
|
|
69
69
|
|
|
70
70
|
Install `scip-cli` and run it. On first index, scip-cli will:
|
|
71
71
|
|
|
72
|
-
- Download `scip-typescript` / `scip-python` via `npx` when not already on PATH
|
|
72
|
+
- Download `scip-typescript` / `scip-python` via `npx`, `scip-go` via `go install`, or `rust-analyzer` via `rustup` when not already on PATH
|
|
73
73
|
- Download the `scip` converter binary from [GitHub releases](https://github.com/scip-code/scip/releases) into `~/.cache/scip-cli/bin/` when not already on PATH
|
|
74
74
|
- Walk the repo for `tsconfig*.json` project roots (TypeScript monorepos), run `scip-typescript` per project (parallel by default), convert each partial index, then merge into one `index.db`
|
|
75
75
|
|
|
@@ -86,6 +86,12 @@ npm install -g @sourcegraph/scip-typescript
|
|
|
86
86
|
# Python indexer
|
|
87
87
|
npm install -g @sourcegraph/scip-python
|
|
88
88
|
|
|
89
|
+
# Go indexer
|
|
90
|
+
go install github.com/scip-code/scip-go/cmd/scip-go@latest
|
|
91
|
+
|
|
92
|
+
# Rust indexer (rust-analyzer with SCIP support)
|
|
93
|
+
rustup component add rust-analyzer
|
|
94
|
+
|
|
89
95
|
# SCIP CLI for index conversion (GitHub release — not on npm)
|
|
90
96
|
# https://github.com/scip-code/scip/releases (v0.8.1+ recommended)
|
|
91
97
|
```
|
|
@@ -179,9 +185,9 @@ scip-cli deps greet --paths-only | sort -u
|
|
|
179
185
|
|
|
180
186
|
## How It Works
|
|
181
187
|
|
|
182
|
-
1. On first query, automatically detects project language from `package.json` (TS/JS)
|
|
188
|
+
1. On first query, automatically detects project language from `package.json` (TS/JS), `pyproject.toml`/`setup.py` (Python), `go.mod` (Go), or `Cargo.toml` (Rust)
|
|
183
189
|
2. For TypeScript monorepos, walks the repository for `tsconfig*.json` project roots (nested ancestors deduped; root included only when its `include` is broad)
|
|
184
|
-
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial),
|
|
190
|
+
3. Runs `scip-typescript` per project (in parallel when there are multiple projects; set `SCIP_CLI_INDEX_WORKERS=1` to force serial), `scip-python` for Python, `scip-go` for Go, or `rust-analyzer scip` for Rust
|
|
185
191
|
4. Converts each SCIP output to SQLite with `scip expt-convert`, then merges partial databases when needed
|
|
186
192
|
5. Caches the result in `~/.cache/scip-cli/projects/<dirname>-<hash>/index.db` (e.g. `my-monorepo-1a3f7a`)
|
|
187
193
|
6. Subsequent queries are SQLite lookups against that cache (not re-indexing)
|
|
@@ -198,11 +204,23 @@ Optional `.scip-cli.json` in the project root:
|
|
|
198
204
|
}
|
|
199
205
|
```
|
|
200
206
|
|
|
201
|
-
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage.
|
|
207
|
+
- `maxHeapMb` — Node heap for `scip-typescript` / `scip-python` (default **8192 MB** when omitted). Overridden by `SCIP_CLI_MAX_HEAP_MB`. This is the V8 heap cap, not total RAM usage. Does not affect `scip-go`.
|
|
202
208
|
- `indexRoots` — extra TypeScript project directories to include on **first index**, merged with auto-discovered projects.
|
|
203
209
|
- `onlyIndexRoots` — skip auto-discovery and index **only** `indexRoots` (smaller initial index when you only care about part of a monorepo).
|
|
204
210
|
|
|
205
|
-
`SCIP_CLI_INDEX_WORKERS` controls parallel
|
|
211
|
+
`SCIP_CLI_INDEX_WORKERS` controls parallel indexer runs during first index (default: up to 8). Merge into one database is always serial.
|
|
212
|
+
|
|
213
|
+
Other environment variables:
|
|
214
|
+
|
|
215
|
+
| Variable | Purpose |
|
|
216
|
+
| ------------------------------ | --------------------------------------------------------------------------------------------- |
|
|
217
|
+
| `SCIP_CLI_MAX_HEAP_MB` | Node heap for `scip-typescript` / `scip-python` (overrides `maxHeapMb` in config) |
|
|
218
|
+
| `SCIP_CLI_TS_INDEX_BATCH_SIZE` | Split large TS repos into multiple `scip-typescript` runs (default: all tsconfigs in one run) |
|
|
219
|
+
| `SCIP_CLI_MERGE_BATCH_SIZE` | SQLite ATTACH batch size when merging part DBs (max 9) |
|
|
220
|
+
| `SCIP_CLI_MAX_DEF_LINES` | Max definition lines in `code` output |
|
|
221
|
+
| `SCIP_CLI_DEBUG` | Log SQL queries to stderr |
|
|
222
|
+
|
|
223
|
+
**Version policy:** only the `scip` converter (`expt-convert`) is pinned to the 0.8.x release line because it defines the SQLite schema. Language indexers (`scip-typescript`, `scip-python` via `npx`; `scip-go` via `go install @latest`) install at latest on first use. `rust-analyzer` installs via `rustup component add`.
|
|
206
224
|
|
|
207
225
|
Large monorepos (>10 tsconfig projects) log per-project progress to stderr during indexing; smaller repos stay quiet aside from the final `Indexed … (size)` line.
|
|
208
226
|
|
|
@@ -213,7 +231,7 @@ scip-cli reindex --path packages/server
|
|
|
213
231
|
scip-cli reindex --path packages/api --path packages/worker
|
|
214
232
|
```
|
|
215
233
|
|
|
216
|
-
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
234
|
+
`--path` limits which discovered tsconfig projects are indexed (prefix match, same idea as query `--path`). **TypeScript only** — other languages reject `reindex --path`. The scope is saved as `index-scope.json` next to `index.db` and reused until you run a full `scip-cli reindex` with no `--path`.
|
|
217
235
|
|
|
218
236
|
Run `scip-cli reindex` after changing scope, `.scip-cli.json` index settings, or when you want a fresh index.
|
|
219
237
|
|
|
@@ -230,24 +248,24 @@ scip-cli analyze --priority high --limit 25 # dead exports & cycles only
|
|
|
230
248
|
|
|
231
249
|
Sections are tagged `[high]`, `[medium]`, `[low]` and listed in that order.
|
|
232
250
|
|
|
233
|
-
| Tier
|
|
234
|
-
|
|
|
235
|
-
| **high**
|
|
236
|
-
| **medium** | Same-file only, change surface (file target)
|
|
237
|
-
| **low**
|
|
251
|
+
| Tier | Project sections | Action |
|
|
252
|
+
| ---------- | ---------------------------------------------------- | -------------------------------------------------------------------- |
|
|
253
|
+
| **high** | Cycles, unreferenced, dead exports, stale types | Nuke or fix cycles; delete unused; `_` prefix |
|
|
254
|
+
| **medium** | Same-file only, change surface (file target) | Module-private by usage |
|
|
255
|
+
| **low** | Test-only consumers, coupling, bottlenecks, hotspots | Noisy on Python (index omits many same-file calls); verify with `rg` |
|
|
238
256
|
|
|
239
257
|
Use `--priority high` for a quick gate; `--priority high,medium` adds context. File drill-down adds change surface and unused imports.
|
|
240
258
|
|
|
241
259
|
**What to look at first**
|
|
242
260
|
|
|
243
|
-
| Section
|
|
244
|
-
|
|
|
245
|
-
| **Cycles**
|
|
246
|
-
| **Unreferenced**
|
|
247
|
-
| **Dead exports**
|
|
248
|
-
| **Stale types**
|
|
249
|
-
| **Same-file only**
|
|
250
|
-
| **Test-only consumers** | Cross-file refs are all from tests — promote to e2e or accept as internal
|
|
261
|
+
| Section | Easy pickings |
|
|
262
|
+
| ----------------------- | ------------------------------------------------------------------------------------------------------ |
|
|
263
|
+
| **Cycles** | Import/mention cycles between production files — break the edge or extract shared code |
|
|
264
|
+
| **Unreferenced** | No usage in the index at all — delete |
|
|
265
|
+
| **Dead exports** | No external refs — delete or `_` prefix |
|
|
266
|
+
| **Stale types** | Classes/types with no external consumer in the index — verify in-file or type-only use before removing |
|
|
267
|
+
| **Same-file only** | Used only inside defining file — rename to `_` |
|
|
268
|
+
| **Test-only consumers** | Cross-file refs are all from tests — promote to e2e or accept as internal |
|
|
251
269
|
|
|
252
270
|
**Per-file or package drill-down** on hubs or suspects:
|
|
253
271
|
|
|
@@ -256,7 +274,7 @@ scip-cli analyze scip_cli/queries.py --limit 20 # file: scoped project + per-f
|
|
|
256
274
|
scip-cli analyze scip_cli --limit 15 # directory: scoped project + each file under it
|
|
257
275
|
```
|
|
258
276
|
|
|
259
|
-
`Dead exports in file` lists same-module symbols with no
|
|
277
|
+
`Dead exports in file` lists same-module symbols with no _external_ refs — module-private `_helpers` are filtered out. Remaining rows are worth a manual `rg` check.
|
|
260
278
|
|
|
261
279
|
**Defaults:** project-wide and directory analyze skip `tests/`, `*.test.*`, `*.spec.*`, `conftest.py`, and `__tests__/`. Pass `--include-tests` to include them. File-target analyze always includes that file.
|
|
262
280
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: scip-cli
|
|
3
|
-
description: Read when needing symbols, definitions, refs, members, or SQL health dashboards in TS/JS or
|
|
3
|
+
description: Read when needing symbols, definitions, refs, members, or SQL health dashboards in TS/JS, Python, Go, or Rust
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
TypeScript/JavaScript (.ts, .tsx, .js, .jsx)
|
|
6
|
+
TypeScript/JavaScript (.ts, .tsx, .js, .jsx), Python (.py), Go (.go), and Rust (.rs) — not GraphQL, CSS, or other files.
|
|
7
7
|
|
|
8
8
|
All commands are sub-commands of `scip-cli`. Run from the project root.
|
|
9
9
|
|
|
@@ -27,10 +27,10 @@ All commands are sub-commands of `scip-cli`. Run from the project root.
|
|
|
27
27
|
- **Bare names** resolve functions, types (aliases + interfaces), and classes. Use dotted qualifiers to disambiguate members: `code Widget.run`, `refs Foo.setBar`, `search MyClass.myMethod`, `members pkg.MyClass`. Type/object fields use the same form: `search Options.verbose`, `code Options.verbose`. Consts/let/var are not kept in the index (too many rows, single-line defs) — use `rg` or read the file. Class methods need `members ClassName`, not bare `code methodName`.
|
|
28
28
|
- **Ambiguous types** (e.g. `Opts` in multiple hooks) — `code`/`refs` return all matches up to `--limit`; `members` and `analyze` pick the first match with a stderr warning. Use dotted qualifiers or `--path` to narrow.
|
|
29
29
|
- **Stale index** — the cache is a snapshot; run `scip-cli reindex` after substantive code changes (no automatic invalidation).
|
|
30
|
-
- **Query `--path` vs `reindex --path`** — query `--path` filters results only. `reindex --path` persists scope and **replaces** the cached index with only those tsconfig projects; run full `reindex` (no `--path`) to restore.
|
|
30
|
+
- **Query `--path` vs `reindex --path`** — query `--path` filters results only. `reindex --path` is **TypeScript-only**: persists scope and **replaces** the cached index with only those tsconfig projects; run full `reindex` (no `--path`) to restore.
|
|
31
31
|
- **First run** in a project may auto-index (one-time wait; large monorepos with many `tsconfig.json` files take longer). Projects index in parallel by default (`SCIP_CLI_INDEX_WORKERS`; merge is serial). Repos with more than 10 tsconfig projects log per-project progress to stderr. JS-only projects (no `tsconfig.json`) are supported automatically.
|
|
32
32
|
- **Monorepos** are indexed by walking for `tsconfig*.json` under the repo (skips `node_modules`, `.git`, etc.). Nested parent/child projects are deduped. Add extra roots or limit indexing with `.scip-cli.json` (see README). Use query `--path packages/api` to scope lookups.
|
|
33
|
-
- **Prerequisites**: Node.js (for `npx`
|
|
33
|
+
- **Prerequisites**: Node.js (for TypeScript/Python via `npx`), Go toolchain (for Go via `go install`), or Rust toolchain (for Rust via `rustup`). The `scip` converter auto-downloads on first use if missing; `scip-typescript` / `scip-python` download via `npx`; `scip-go` downloads via `go install` to `~/go/bin`; `rust-analyzer` installs via `rustup component add`. Optional `.scip-cli.json` for extra index roots or heap tuning. `brew install scip` installs an unrelated optimization solver — scip-cli ignores it and downloads the real binary.
|
|
34
34
|
|
|
35
35
|
## Details
|
|
36
36
|
|
|
@@ -155,6 +155,21 @@ def has_same_file_reference_usage(db, symbol_id: int, def_doc_id: int) -> bool:
|
|
|
155
155
|
return row is not None
|
|
156
156
|
|
|
157
157
|
|
|
158
|
+
def has_same_file_usage_mention(db, symbol_id: int, def_doc_id: int) -> bool:
|
|
159
|
+
"""Non-definition mention in the defining file (in-file callers)."""
|
|
160
|
+
row = _fetch_one(
|
|
161
|
+
db,
|
|
162
|
+
"""
|
|
163
|
+
SELECT 1 FROM mentions m
|
|
164
|
+
JOIN chunks c ON m.chunk_id = c.id
|
|
165
|
+
WHERE m.symbol_id = ? AND c.document_id = ? AND m.role != 1
|
|
166
|
+
LIMIT 1
|
|
167
|
+
""",
|
|
168
|
+
(symbol_id, def_doc_id),
|
|
169
|
+
)
|
|
170
|
+
return row is not None
|
|
171
|
+
|
|
172
|
+
|
|
158
173
|
def file_has_scip_importers(db, relative_path: str, *, live: LiveIndex, def_doc_id: int) -> bool:
|
|
159
174
|
"""True when the index shows another file importing this module or its symbols."""
|
|
160
175
|
if def_doc_id in live.live_module_docs:
|
|
@@ -18,7 +18,12 @@ from .common import (
|
|
|
18
18
|
stale_type_noise,
|
|
19
19
|
)
|
|
20
20
|
from .graph import FILE_EDGES_SQL, fetch_file_edges, find_longer_cycles
|
|
21
|
-
from .live import
|
|
21
|
+
from .live import (
|
|
22
|
+
LiveIndex,
|
|
23
|
+
file_has_scip_importers,
|
|
24
|
+
has_same_file_reference_usage,
|
|
25
|
+
has_same_file_usage_mention,
|
|
26
|
+
)
|
|
22
27
|
from .sections import Check, Priority, run_checks
|
|
23
28
|
|
|
24
29
|
|
|
@@ -165,6 +170,8 @@ def _format_dead_export_rows(
|
|
|
165
170
|
continue
|
|
166
171
|
if has_same_file_reference_usage(db, symbol_id, def_doc_id):
|
|
167
172
|
continue
|
|
173
|
+
if has_same_file_usage_mention(db, symbol_id, def_doc_id):
|
|
174
|
+
continue
|
|
168
175
|
if live.dead_export_noise(symbol, def_doc_id):
|
|
169
176
|
continue
|
|
170
177
|
lines.append(f"{short_name(symbol)} loc={loc} ({path})")
|
|
@@ -22,7 +22,7 @@ def main(args):
|
|
|
22
22
|
sys.exit(1)
|
|
23
23
|
|
|
24
24
|
path_args = getattr(args, "path", None) or []
|
|
25
|
-
if path_args and lang
|
|
25
|
+
if path_args and lang != Language.TYPESCRIPT:
|
|
26
26
|
print("Error: reindex --path is only supported for TypeScript projects", file=sys.stderr)
|
|
27
27
|
sys.exit(1)
|
|
28
28
|
|
|
@@ -19,6 +19,8 @@ _SKIP_DIR_NAMES = frozenset(
|
|
|
19
19
|
".turbo",
|
|
20
20
|
".nx",
|
|
21
21
|
"tmp",
|
|
22
|
+
".venv",
|
|
23
|
+
"venv",
|
|
22
24
|
}
|
|
23
25
|
)
|
|
24
26
|
|
|
@@ -121,3 +123,55 @@ def should_index_root_alongside_projects(root: Path, projects: list[Path]) -> bo
|
|
|
121
123
|
if not root_tsconfig.is_file():
|
|
122
124
|
return False
|
|
123
125
|
return _tsconfig_covers_subdirectories(root_tsconfig)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _walk_marker_files(root: Path, marker_names: frozenset[str]) -> list[Path]:
|
|
129
|
+
"""Return directories containing any of marker_names (relative to root)."""
|
|
130
|
+
root = root.resolve()
|
|
131
|
+
found: list[Path] = []
|
|
132
|
+
|
|
133
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
134
|
+
dirnames[:] = [name for name in dirnames if name not in _SKIP_DIR_NAMES and not name.startswith(".")]
|
|
135
|
+
if not any(name in marker_names for name in filenames):
|
|
136
|
+
continue
|
|
137
|
+
project_dir = Path(dirpath).resolve()
|
|
138
|
+
try:
|
|
139
|
+
relative = project_dir.relative_to(root)
|
|
140
|
+
except ValueError:
|
|
141
|
+
continue
|
|
142
|
+
found.append(relative if relative.parts else Path("."))
|
|
143
|
+
|
|
144
|
+
return found
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def discover_golang_modules(root: Path) -> list[Path]:
|
|
148
|
+
"""Return Go module directories (go.mod roots) under the repository."""
|
|
149
|
+
root = root.resolve()
|
|
150
|
+
modules = _walk_marker_files(root, frozenset({"go.mod"}))
|
|
151
|
+
if modules:
|
|
152
|
+
return sorted(set(modules), key=str)
|
|
153
|
+
if (root / "go.mod").is_file():
|
|
154
|
+
return [Path(".")]
|
|
155
|
+
return [Path(".")]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def discover_python_projects(root: Path) -> list[Path]:
|
|
159
|
+
"""Return Python package roots (pyproject.toml or setup.py) under the repository."""
|
|
160
|
+
root = root.resolve()
|
|
161
|
+
projects = _walk_marker_files(root, frozenset({"pyproject.toml", "setup.py"}))
|
|
162
|
+
if projects:
|
|
163
|
+
return sorted(set(projects), key=str)
|
|
164
|
+
if (root / "pyproject.toml").is_file() or (root / "setup.py").is_file():
|
|
165
|
+
return [Path(".")]
|
|
166
|
+
return [Path(".")]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def discover_rust_crates(root: Path) -> list[Path]:
|
|
170
|
+
"""Return Rust crate directories (Cargo.toml roots) under the repository."""
|
|
171
|
+
root = root.resolve()
|
|
172
|
+
crates = _walk_marker_files(root, frozenset({"Cargo.toml"}))
|
|
173
|
+
if crates:
|
|
174
|
+
return sorted(set(crates), key=str)
|
|
175
|
+
if (root / "Cargo.toml").is_file():
|
|
176
|
+
return [Path(".")]
|
|
177
|
+
return [Path(".")]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""SCIP index building and SQLite database access."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from ..merge import merge_sqlite_indexes
|
|
6
|
+
from .constants import (
|
|
7
|
+
DEFAULT_MAX_HEAP_MB,
|
|
8
|
+
DEFAULT_TS_INDEX_BATCH_SIZE,
|
|
9
|
+
MAX_TS_INDEX_BATCH_SIZE,
|
|
10
|
+
)
|
|
11
|
+
from .core import format_db_size, get_db, index_project, indexer_env, log_index_complete
|
|
12
|
+
from .orchestrate import ts_index_batch_size
|
|
13
|
+
from .runners import run_indexer_with_fallback
|
|
14
|
+
from .typescript import typescript_projects
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"DEFAULT_MAX_HEAP_MB",
|
|
18
|
+
"DEFAULT_TS_INDEX_BATCH_SIZE",
|
|
19
|
+
"MAX_TS_INDEX_BATCH_SIZE",
|
|
20
|
+
"format_db_size",
|
|
21
|
+
"get_db",
|
|
22
|
+
"index_project",
|
|
23
|
+
"indexer_env",
|
|
24
|
+
"log_index_complete",
|
|
25
|
+
"merge_sqlite_indexes",
|
|
26
|
+
"run_indexer_with_fallback",
|
|
27
|
+
"ts_index_batch_size",
|
|
28
|
+
"typescript_projects",
|
|
29
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Indexing constants."""
|
|
2
|
+
|
|
3
|
+
INDEX_TIMEOUT = 300
|
|
4
|
+
DEFAULT_MAX_HEAP_MB = 8192
|
|
5
|
+
PROGRESS_LOG_MIN_PROJECTS = 10
|
|
6
|
+
SCIP_INSTALL_URL = "https://github.com/scip-code/scip/releases"
|
|
7
|
+
|
|
8
|
+
# Language indexers (npx / go install) use latest on first use.
|
|
9
|
+
# The scip expt-convert binary is pinned separately in scip_tool.py (DB schema).
|
|
10
|
+
# scip-typescript accepts many tsconfig paths per invocation (one .scip, one convert, no merge).
|
|
11
|
+
# Split only when SCIP_CLI_TS_INDEX_BATCH_SIZE is set (OOM/timeout tuning).
|
|
12
|
+
DEFAULT_TS_INDEX_BATCH_SIZE = None
|
|
13
|
+
MAX_TS_INDEX_BATCH_SIZE = 2_147_483_647
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""SCIP protobuf to SQLite conversion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from ..scip_tool import ensure_scip_binary
|
|
12
|
+
from .constants import SCIP_INSTALL_URL
|
|
13
|
+
from .postprocess import postprocess_index
|
|
14
|
+
from .runners import run_subprocess
|
|
15
|
+
|
|
16
|
+
_scip_version_warned = False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _scip_version(binary):
|
|
20
|
+
result = run_subprocess([binary, "--version"], cwd=os.getcwd())
|
|
21
|
+
if result.returncode != 0:
|
|
22
|
+
return None
|
|
23
|
+
match = re.search(r"v?(\d+)\.(\d+)\.(\d+)", result.stdout + result.stderr)
|
|
24
|
+
if not match:
|
|
25
|
+
return None
|
|
26
|
+
return tuple(int(part) for part in match.groups())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def resolve_scip_binary():
|
|
30
|
+
"""Locate the scip CLI used to convert protobuf indexes to SQLite."""
|
|
31
|
+
return str(ensure_scip_binary())
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _warn_old_scip(binary):
|
|
35
|
+
global _scip_version_warned
|
|
36
|
+
if _scip_version_warned:
|
|
37
|
+
return
|
|
38
|
+
version = _scip_version(binary)
|
|
39
|
+
if version and version < (0, 8, 0):
|
|
40
|
+
_scip_version_warned = True
|
|
41
|
+
print(
|
|
42
|
+
f"Warning: {binary} {'.'.join(map(str, version))} is older than 0.8.0; "
|
|
43
|
+
+ "upgrade from "
|
|
44
|
+
+ f"{SCIP_INSTALL_URL} if indexing fails.",
|
|
45
|
+
file=sys.stderr,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def project_path_prefix(project: Path | str | None) -> str | None:
|
|
50
|
+
if project is None:
|
|
51
|
+
return None
|
|
52
|
+
path = Path(project)
|
|
53
|
+
if path == Path("."):
|
|
54
|
+
return None
|
|
55
|
+
return path.as_posix()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def prefix_document_paths(db_path: Path, prefix: str) -> None:
|
|
59
|
+
"""Rewrite document paths to be relative to the repository root."""
|
|
60
|
+
conn = sqlite3.connect(str(db_path))
|
|
61
|
+
try:
|
|
62
|
+
conn.execute(
|
|
63
|
+
"UPDATE documents SET relative_path = ? || '/' || relative_path",
|
|
64
|
+
(prefix,),
|
|
65
|
+
)
|
|
66
|
+
conn.commit()
|
|
67
|
+
finally:
|
|
68
|
+
conn.close()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def convert_scip_to_db(scip_path, db_path, *, document_path_prefix: Path | str | None = None):
|
|
72
|
+
"""Convert a SCIP protobuf file to a SQLite index at db_path."""
|
|
73
|
+
db_path = Path(db_path)
|
|
74
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
75
|
+
if db_path.exists():
|
|
76
|
+
db_path.unlink()
|
|
77
|
+
|
|
78
|
+
scip_binary = resolve_scip_binary()
|
|
79
|
+
_warn_old_scip(scip_binary)
|
|
80
|
+
|
|
81
|
+
result = run_subprocess(
|
|
82
|
+
[scip_binary, "expt-convert", str(scip_path), "--output", db_path.name],
|
|
83
|
+
cwd=str(db_path.parent),
|
|
84
|
+
)
|
|
85
|
+
if result.returncode != 0:
|
|
86
|
+
print(result.stderr, file=sys.stderr)
|
|
87
|
+
raise RuntimeError("Failed to convert index")
|
|
88
|
+
if not db_path.exists():
|
|
89
|
+
raise RuntimeError("Failed to convert index")
|
|
90
|
+
|
|
91
|
+
postprocess_index(db_path)
|
|
92
|
+
prefix = project_path_prefix(document_path_prefix)
|
|
93
|
+
if prefix is not None:
|
|
94
|
+
prefix_document_paths(db_path, prefix)
|