local-codebase-indexer 0.6.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/AGENTS.md +9 -0
  2. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/CHANGELOG.md +13 -0
  3. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/MANIFEST.in +2 -1
  4. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/PKG-INFO +62 -2
  5. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/README.md +61 -1
  6. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/README.zh-CN.md +10 -0
  7. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/README.zh-TW.md +23 -0
  8. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/TODO.md +2 -0
  9. local_codebase_indexer-0.7.0/benchmarks/queries_project.json +21 -0
  10. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/__init__.py +1 -1
  11. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/cli.py +35 -1
  12. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/core.py +27 -2
  13. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/db.py +34 -7
  14. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/embedder.py +45 -2
  15. local_codebase_indexer-0.7.0/local_codebase_indexer/evaluation.py +149 -0
  16. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/impact.py +47 -19
  17. local_codebase_indexer-0.7.0/local_codebase_indexer/locking.py +74 -0
  18. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/mcp_server.py +4 -1
  19. local_codebase_indexer-0.7.0/local_codebase_indexer/pagination.py +37 -0
  20. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/query.py +143 -49
  21. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/synonyms.py +50 -6
  22. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/PKG-INFO +62 -2
  23. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/SOURCES.txt +8 -0
  24. local_codebase_indexer-0.7.0/scripts/release_assets.py +133 -0
  25. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/server.json +2 -2
  26. local_codebase_indexer-0.7.0/tests/test_evaluation.py +88 -0
  27. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/tests/test_indexer.py +5 -0
  28. local_codebase_indexer-0.7.0/tests/test_navigation_stability.py +256 -0
  29. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/tests/test_onnx_plumbing.py +59 -0
  30. local_codebase_indexer-0.7.0/tests/test_release_assets.py +94 -0
  31. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/.claude-plugin/marketplace.json +0 -0
  32. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/CLAUDE.md +0 -0
  33. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/README.md +0 -0
  34. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/baseline.json +0 -0
  35. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Auth/LoginService.cs +0 -0
  36. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Controllers/LotController.cs +0 -0
  37. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Controllers/ScrapController.cs +0 -0
  38. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Data/MesDbContext.cs +0 -0
  39. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Legacy/frmChangeLine.frm +0 -0
  40. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Models/DrillBreakEvent.cs +0 -0
  41. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Models/Lot.cs +0 -0
  42. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Models/ScrapRecord.cs +0 -0
  43. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Printing/LabelPrinter.cs +0 -0
  44. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Repositories/LotRepository.cs +0 -0
  45. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Repositories/ScrapRepository.cs +0 -0
  46. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Services/IScrapService.cs +0 -0
  47. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Services/InspectionService.cs +0 -0
  48. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Services/LotService.cs +0 -0
  49. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Services/ScrapService.cs +0 -0
  50. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Sql/MES_Lot.sql +0 -0
  51. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Sql/usp_Inspection_Report.sql +0 -0
  52. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Sql/usp_Lot_UpdateStatus.sql +0 -0
  53. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Sql/usp_Scrap_Insert.sql +0 -0
  54. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Utils/DateRangeHelper.cs +0 -0
  55. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Utils/PartNumberDecoder.cs +0 -0
  56. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Web/scrapForm.ts +0 -0
  57. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/Web.config +0 -0
  58. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/corpus/tools/export_report.py +0 -0
  59. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/queries_stdlib.json +0 -0
  60. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/queries_synthetic.json +0 -0
  61. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/benchmarks/run_benchmark.py +0 -0
  62. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/bin/indexer +0 -0
  63. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/index.md +0 -0
  64. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/llms.txt +0 -0
  65. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/__main__.py +0 -0
  66. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/callgraph.py +0 -0
  67. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/cscalls.py +0 -0
  68. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/csdata.py +0 -0
  69. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/dataflow.py +0 -0
  70. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/langparse.py +0 -0
  71. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/scanner.py +0 -0
  72. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer/sqlaccess.py +0 -0
  73. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/dependency_links.txt +0 -0
  74. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/entry_points.txt +0 -0
  75. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/requires.txt +0 -0
  76. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/local_codebase_indexer.egg-info/top_level.txt +0 -0
  77. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/plugins/local-codebase-indexer/.claude-plugin/plugin.json +0 -0
  78. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/plugins/local-codebase-indexer/.mcp.json +0 -0
  79. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/pyproject.toml +0 -0
  80. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/requirements.txt +0 -0
  81. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/setup.cfg +0 -0
  82. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/tests/test_dataaccess.py +0 -0
  83. {local_codebase_indexer-0.6.0 → local_codebase_indexer-0.7.0}/tests/test_reliability.py +0 -0
@@ -64,3 +64,12 @@ The package is `local_codebase_indexer/`. See `README.md` for design notes and k
64
64
  - `data-access <table> --confidence confirmed --limit 20 --offset 20` pages each section independently. JSON has section totals and next offsets; `--full` really lists every entry.
65
65
  - `impact` includes historical accesses and callers for removed code. `snapshot=before` line numbers refer to the base revision. Historical indexes are cached in `.ai_context/impact/`; deleting that cache is safe.
66
66
  - A missing root is an error; an empty or failed indexing run returns exit status 3. `--json` and `--root` work before or after the subcommand.
67
+
68
+ ## Development improvements (0.7.0)
69
+
70
+ - Prefer `search <query> --cursor <pagination.next_cursor>` for subsequent pages; retain query/filters. Cursor errors require a new search. `--explain` / MCP `explain=true` adds matched terms, expansion sources, layer ranks and SQL evidence.
71
+ - `evaluate --queries .ai_context/eval.json --save .ai_context/report.json` measures your own verified navigation questions. Optional recall/latency/token budgets fail with exit status 3. Do not change gold answers or lower thresholds merely to make a run pass.
72
+ - Exit status 4 means the index is busy; retry after the other process finishes. Configure `INDEXER_LOCK_TIMEOUT` and `INDEXER_BUSY_TIMEOUT` only when needed. Do not delete active lock files.
73
+ - `impact A..B` and `A...B` use both Git snapshots and label target lines `after`; a single base revision compares with the working tree.
74
+ - Context clipping keeps focus lines. Synonym configuration hot-reloads. ONNX identities include model/tokenizer/configuration fingerprints.
75
+ - Future Release events validate and build once, then use the same files for PyPI and GitHub attachments. Validate locally with `python scripts/release_assets.py --tag v<package-version> --dist <directory>`; add `--upload` only when publishing was requested.
@@ -1,5 +1,18 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.7.0
4
+
5
+ - Stable search ranking across page sizes/offsets, with query- and revision-bound continuation cursors.
6
+ - Historical Git ranges analyze both requested snapshots instead of mixing in the current working tree.
7
+ - ONNX identities fingerprint model/tokenizer/external weights and inference settings, triggering rebuilds when contents change.
8
+ - Process-safe indexing, consistent query snapshots, bounded busy responses (CLI exit 4), and interrupted-batch recovery.
9
+ - Context clipping preserves focus lines and reports omitted/clipped content.
10
+ - Project synonyms reload on queries; malformed settings warn once per content version and recover after correction.
11
+ - Opt-in search explanations show matched terms, expansion provenance, layer ranks/contributions and SQL evidence.
12
+ - Project evaluation datasets, per-query reports, retrieval/latency/token metrics, baseline comparison and enforceable budgets.
13
+ - Future Release events test and build once, validate versions, and upload the same artifacts to PyPI/GitHub with SHA256SUMS. Upload retries are idempotent and do not overwrite conflicting assets.
14
+
15
+
3
16
  ## 0.6.0
4
17
 
5
18
  ### Fixed
@@ -1,9 +1,10 @@
1
1
  include server.json CHANGELOG.md README*.md TODO.md AGENTS.md CLAUDE.md llms.txt index.md requirements.txt
2
+ graft scripts
2
3
  graft tests
3
4
  graft plugins
4
5
  graft .claude-plugin
5
6
  graft benchmarks
6
7
  prune benchmarks/__pycache__
7
- global-exclude __pycache__ *.pyc
8
+ global-exclude __pycache__ *.py[cod]
8
9
  prune benchmarks/corpus/.ai_context
9
10
  include bin/indexer
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: local-codebase-indexer
3
- Version: 0.6.0
3
+ Version: 0.7.0
4
4
  Summary: Local codebase indexer and context provider for AI coding assistants: MCP server and CLI with SQL data-access and C# call relations. No cloud, no server.
5
5
  Author: sfy257000
6
6
  Project-URL: Homepage, https://github.com/sfy257000/local-codebase-indexer
@@ -103,7 +103,7 @@ local-codebase-indexer --root <project> data-access MES_Lot --full
103
103
 
104
104
  `doctor` reports supported text files, ignore rules, excluded directories, unsupported extensions, files over 5 MB, binary files and index health. `--path` explains a specific file, including a parent directory that prevents indexing. Directory counts are counts of excluded directories, not all the files hidden inside them. Nested `.gitignore` negations override parent file patterns; an excluded parent directory must first be re-included, just as with Git. Built-in excluded directories such as `node_modules` remain excluded.
105
105
 
106
- Search filters apply **before** ranking. `--file` is a case-sensitive glob over the complete project-relative path (`*` also matches `/`); languages are `csharp`, `python`, `javascript`, `typescript`, `tsx`, and `text`. The JSON `pagination` object has `has_more` and `next_offset`; keep the same query and filters for subsequent pages. Pages assume the index remains unchanged. No-result responses suggest similar symbols where available and explain useful next steps. General verb paraphrases expand keyword/symbol matching without diluting vector queries; project-specific jargon still belongs in `.ai_context/synonyms.json`.
106
+ Search filters apply **before** ranking. `--file` is a case-sensitive glob over the complete project-relative path (`*` also matches `/`); languages are `csharp`, `python`, `javascript`, `typescript`, `tsx`, and `text`. The JSON `pagination` object has `has_more` and `next_offset`; keep the same query and filters for subsequent pages. Offset pages assume the index remains unchanged; cursor pages reject a changed index or configuration. No-result responses suggest similar symbols where available and explain useful next steps. General verb paraphrases expand keyword/symbol matching without diluting vector queries; project-specific jargon still belongs in `.ai_context/synonyms.json`.
107
107
 
108
108
  Data-access `--confidence` selects an exact level (`confirmed`, `probable`, or `uncertain`). `--limit` and `--offset` apply independently to each section, including programs and procedure accesses. JSON includes `pagination.sections` with totals and next offsets. MCP accepts the same `limit`, `offset`, and `confidence` fields; search uses `file_pattern` and `language`. `--full` / MCP `detail=true` lists all entries when no explicit page size is supplied. The old 40-entry display and 200-entry query caps have been removed.
109
109
 
@@ -111,6 +111,66 @@ Data-access `--confidence` selects an exact level (`confirmed`, `probable`, or `
111
111
 
112
112
  A nonexistent `--root` is rejected without creating it. `index` returns exit status 3 for an empty or partially failed run; use `doctor` and `status` to diagnose it. Invalid MCP inputs return an error while keeping the server available for the next request.
113
113
 
114
+ ## Stable navigation and search explanations
115
+
116
+ Search ranking is independent of `--limit` and `--offset`. To continue a search safely, use the opaque `pagination.next_cursor` returned in JSON, or the cursor printed by the text/MCP response:
117
+
118
+ ```bash
119
+ local-codebase-indexer --root <project> search "retrieve invoice" --limit 5 --explain --json
120
+ local-codebase-indexer --root <project> search "retrieve invoice" --limit 5 --cursor '<next_cursor>' --json
121
+ ```
122
+
123
+ Keep the same query, file glob and language when using a cursor; the page size may change. Cursors are bound to a completed index revision, the embedding model, and synonym settings. A no-op refresh keeps them valid; a changed index or configuration rejects the cursor with an instruction to start again. `--offset` remains supported but has no revision guard. MCP search accepts `cursor` and `explain` too.
124
+
125
+ `--explain` adds matched query terms, matched synonym expansions with their built-in/project/mined-comment sources, and each retrieval layer's rank and RRF contribution. Data-access hits also carry SQL evidence, confidence, access mechanism and procedure chains. Existing `relations` output includes caller resolution reasons and confidence. These are static-analysis/retrieval explanations, not proof that a runtime path executes or a score is a probability. Explanations are omitted by default to keep responses compact.
126
+
127
+ Project synonym settings reload on subsequent queries, including deletion and replacement of the settings file. Invalid settings log a warning and fall back to built-in groups; the same invalid contents are not repeatedly logged. ONNX identities include a streamed content fingerprint of the model directory (including tokenizer and external weights), pooling, dimensions and query prefix. Use a dedicated model directory: changing other files in that directory can also cause a rebuild. Generated `.ai_context`, environment and cache directories are excluded from the fingerprint. Changes to loaded model files are rejected; restart the server to load the updated model. Processes sharing an index must use the same model/settings.
128
+
129
+ Large context requests retain focus lines first, fit neighboring lines within the character budget, and clip individual oversized lines. Returned ranges describe the content actually returned; `omitted_lines` and `truncated_lines` explain clipping. An entirely out-of-file focus request is an error.
130
+
131
+ Git ranges such as `A..B` and `A...B` analyze the actual target commit, even when HEAD or the working tree differs. Range results use `snapshot=after`; `revisions` identifies both commits. A single revision still compares that revision against the working tree. Historical line numbers always belong to their labeled snapshot.
132
+
133
+ ## Evaluate your own project questions
134
+
135
+ Keep questions collected from real navigation work in `.ai_context/eval.json` (ignored by Git and the index):
136
+
137
+ ```json
138
+ {
139
+ "version": 1,
140
+ "queries": [
141
+ {
142
+ "id": "invoice-read",
143
+ "query": "Where do we retrieve an invoice?",
144
+ "language": "python",
145
+ "relevant": [{"file": "src/invoices.py", "symbol": "load_invoice"}]
146
+ }
147
+ ]
148
+ }
149
+ ```
150
+
151
+ Replace the example with verified file paths and optional qualified-symbol suffixes from your project. Each question needs a unique ID and at least one expected answer. `file_pattern` and `language` are optional query filters. Avoid copying question text into the source corpus or authoring only questions that already rank well.
152
+
153
+ ```bash
154
+ local-codebase-indexer --root <project> evaluate --queries <project>/.ai_context/eval.json --save <project>/.ai_context/baseline.json
155
+ local-codebase-indexer --root <project> evaluate --queries <project>/.ai_context/eval.json --baseline <project>/.ai_context/baseline.json --min-recall 0.8 --max-recall-drop 0.02 --max-p95-ms 500 --max-mean-tokens 1500 --save <project>/.ai_context/current.json
156
+ ```
157
+
158
+ The report includes Recall@k, hit rate, MRR, p50/p95 latency, estimated output tokens, per-question answers and the index/model/configuration identity. Indexing time is separate from search latency. Tokens are estimated as `ceil(rendered characters / 4)`, not counted by a specific model tokenizer. Empty/invalid datasets fail; breached quality or performance budgets return exit status 3. Baseline comparisons require the same questions, answers, filters and result limit. Reports cannot overwrite their input questions or baseline.
159
+
160
+ `benchmarks/queries_project.json` supplies 15 curated questions about this repository's actual implementation. They are navigation smoke examples, not collected user queries or an independent quality benchmark. Run them with `local-codebase-indexer --root . evaluate --queries benchmarks/queries_project.json --save .ai_context/project-report.json`; use your own recorded questions for representative quality gates.
161
+
162
+ ## Multiple local processes
163
+
164
+ Index initialization and writes are coordinated with an OS lock. Query reads use a consistent SQLite snapshot and wait for a writer to finish its batches. POSIX readers share a lock; Windows readers serialize. Locks are automatically released on process exit; do not delete `.lock` files while processes are running.
165
+
166
+ `INDEXER_LOCK_TIMEOUT` controls the maximum process-lock wait (default 10 seconds), and `INDEXER_BUSY_TIMEOUT` controls SQLite's wait for writers that do not use the indexer's lock (default 5 seconds). Both accept finite nonnegative seconds. CLI contention returns exit status 4 with a retry hint; MCP reports a tool error while keeping the session available. Interrupted batch indexing is reported as degraded and automatically forces a complete refresh on the next indexing run. Separate `CodebaseTools` instances/connections are required for separate threads.
167
+
168
+ ## Release automation
169
+
170
+ Publishing a future GitHub Release runs tests, builds distributions once, validates package/tag/registry versions, and generates `SHA256SUMS`. PyPI and GitHub attachments receive the same validated artifacts. The attachment job uses GitHub Actions' built-in token with `contents: write`; no personal upload token is needed. Repeating the attachment job skips identical assets and refuses to overwrite conflicting checksums.
171
+
172
+ For local validation without publishing: `python scripts/release_assets.py --tag v<package-version> --dist <build-directory>`. The explicit `--upload --repo owner/name` flags perform uploads. Existing releases are not modified by editing this workflow.
173
+
114
174
  ## Three-layer index
115
175
 
116
176
  | Layer | Implementation |
@@ -68,7 +68,7 @@ local-codebase-indexer --root <project> data-access MES_Lot --full
68
68
 
69
69
  `doctor` reports supported text files, ignore rules, excluded directories, unsupported extensions, files over 5 MB, binary files and index health. `--path` explains a specific file, including a parent directory that prevents indexing. Directory counts are counts of excluded directories, not all the files hidden inside them. Nested `.gitignore` negations override parent file patterns; an excluded parent directory must first be re-included, just as with Git. Built-in excluded directories such as `node_modules` remain excluded.
70
70
 
71
- Search filters apply **before** ranking. `--file` is a case-sensitive glob over the complete project-relative path (`*` also matches `/`); languages are `csharp`, `python`, `javascript`, `typescript`, `tsx`, and `text`. The JSON `pagination` object has `has_more` and `next_offset`; keep the same query and filters for subsequent pages. Pages assume the index remains unchanged. No-result responses suggest similar symbols where available and explain useful next steps. General verb paraphrases expand keyword/symbol matching without diluting vector queries; project-specific jargon still belongs in `.ai_context/synonyms.json`.
71
+ Search filters apply **before** ranking. `--file` is a case-sensitive glob over the complete project-relative path (`*` also matches `/`); languages are `csharp`, `python`, `javascript`, `typescript`, `tsx`, and `text`. The JSON `pagination` object has `has_more` and `next_offset`; keep the same query and filters for subsequent pages. Offset pages assume the index remains unchanged; cursor pages reject a changed index or configuration. No-result responses suggest similar symbols where available and explain useful next steps. General verb paraphrases expand keyword/symbol matching without diluting vector queries; project-specific jargon still belongs in `.ai_context/synonyms.json`.
72
72
 
73
73
  Data-access `--confidence` selects an exact level (`confirmed`, `probable`, or `uncertain`). `--limit` and `--offset` apply independently to each section, including programs and procedure accesses. JSON includes `pagination.sections` with totals and next offsets. MCP accepts the same `limit`, `offset`, and `confidence` fields; search uses `file_pattern` and `language`. `--full` / MCP `detail=true` lists all entries when no explicit page size is supplied. The old 40-entry display and 200-entry query caps have been removed.
74
74
 
@@ -76,6 +76,66 @@ Data-access `--confidence` selects an exact level (`confirmed`, `probable`, or `
76
76
 
77
77
  A nonexistent `--root` is rejected without creating it. `index` returns exit status 3 for an empty or partially failed run; use `doctor` and `status` to diagnose it. Invalid MCP inputs return an error while keeping the server available for the next request.
78
78
 
79
+ ## Stable navigation and search explanations
80
+
81
+ Search ranking is independent of `--limit` and `--offset`. To continue a search safely, use the opaque `pagination.next_cursor` returned in JSON, or the cursor printed by the text/MCP response:
82
+
83
+ ```bash
84
+ local-codebase-indexer --root <project> search "retrieve invoice" --limit 5 --explain --json
85
+ local-codebase-indexer --root <project> search "retrieve invoice" --limit 5 --cursor '<next_cursor>' --json
86
+ ```
87
+
88
+ Keep the same query, file glob and language when using a cursor; the page size may change. Cursors are bound to a completed index revision, the embedding model, and synonym settings. A no-op refresh keeps them valid; a changed index or configuration rejects the cursor with an instruction to start again. `--offset` remains supported but has no revision guard. MCP search accepts `cursor` and `explain` too.
89
+
90
+ `--explain` adds matched query terms, matched synonym expansions with their built-in/project/mined-comment sources, and each retrieval layer's rank and RRF contribution. Data-access hits also carry SQL evidence, confidence, access mechanism and procedure chains. Existing `relations` output includes caller resolution reasons and confidence. These are static-analysis/retrieval explanations, not proof that a runtime path executes or a score is a probability. Explanations are omitted by default to keep responses compact.
91
+
92
+ Project synonym settings reload on subsequent queries, including deletion and replacement of the settings file. Invalid settings log a warning and fall back to built-in groups; the same invalid contents are not repeatedly logged. ONNX identities include a streamed content fingerprint of the model directory (including tokenizer and external weights), pooling, dimensions and query prefix. Use a dedicated model directory: changing other files in that directory can also cause a rebuild. Generated `.ai_context`, environment and cache directories are excluded from the fingerprint. Changes to loaded model files are rejected; restart the server to load the updated model. Processes sharing an index must use the same model/settings.
93
+
94
+ Large context requests retain focus lines first, fit neighboring lines within the character budget, and clip individual oversized lines. Returned ranges describe the content actually returned; `omitted_lines` and `truncated_lines` explain clipping. An entirely out-of-file focus request is an error.
95
+
96
+ Git ranges such as `A..B` and `A...B` analyze the actual target commit, even when HEAD or the working tree differs. Range results use `snapshot=after`; `revisions` identifies both commits. A single revision still compares that revision against the working tree. Historical line numbers always belong to their labeled snapshot.
97
+
98
+ ## Evaluate your own project questions
99
+
100
+ Keep questions collected from real navigation work in `.ai_context/eval.json` (ignored by Git and the index):
101
+
102
+ ```json
103
+ {
104
+ "version": 1,
105
+ "queries": [
106
+ {
107
+ "id": "invoice-read",
108
+ "query": "Where do we retrieve an invoice?",
109
+ "language": "python",
110
+ "relevant": [{"file": "src/invoices.py", "symbol": "load_invoice"}]
111
+ }
112
+ ]
113
+ }
114
+ ```
115
+
116
+ Replace the example with verified file paths and optional qualified-symbol suffixes from your project. Each question needs a unique ID and at least one expected answer. `file_pattern` and `language` are optional query filters. Avoid copying question text into the source corpus or authoring only questions that already rank well.
117
+
118
+ ```bash
119
+ local-codebase-indexer --root <project> evaluate --queries <project>/.ai_context/eval.json --save <project>/.ai_context/baseline.json
120
+ local-codebase-indexer --root <project> evaluate --queries <project>/.ai_context/eval.json --baseline <project>/.ai_context/baseline.json --min-recall 0.8 --max-recall-drop 0.02 --max-p95-ms 500 --max-mean-tokens 1500 --save <project>/.ai_context/current.json
121
+ ```
122
+
123
+ The report includes Recall@k, hit rate, MRR, p50/p95 latency, estimated output tokens, per-question answers and the index/model/configuration identity. Indexing time is separate from search latency. Tokens are estimated as `ceil(rendered characters / 4)`, not counted by a specific model tokenizer. Empty/invalid datasets fail; breached quality or performance budgets return exit status 3. Baseline comparisons require the same questions, answers, filters and result limit. Reports cannot overwrite their input questions or baseline.
124
+
125
+ `benchmarks/queries_project.json` supplies 15 curated questions about this repository's actual implementation. They are navigation smoke examples, not collected user queries or an independent quality benchmark. Run them with `local-codebase-indexer --root . evaluate --queries benchmarks/queries_project.json --save .ai_context/project-report.json`; use your own recorded questions for representative quality gates.
126
+
127
+ ## Multiple local processes
128
+
129
+ Index initialization and writes are coordinated with an OS lock. Query reads use a consistent SQLite snapshot and wait for a writer to finish its batches. POSIX readers share a lock; Windows readers serialize. Locks are automatically released on process exit; do not delete `.lock` files while processes are running.
130
+
131
+ `INDEXER_LOCK_TIMEOUT` controls the maximum process-lock wait (default 10 seconds), and `INDEXER_BUSY_TIMEOUT` controls SQLite's wait for writers that do not use the indexer's lock (default 5 seconds). Both accept finite nonnegative seconds. CLI contention returns exit status 4 with a retry hint; MCP reports a tool error while keeping the session available. Interrupted batch indexing is reported as degraded and automatically forces a complete refresh on the next indexing run. Separate `CodebaseTools` instances/connections are required for separate threads.
132
+
133
+ ## Release automation
134
+
135
+ Publishing a future GitHub Release runs tests, builds distributions once, validates package/tag/registry versions, and generates `SHA256SUMS`. PyPI and GitHub attachments receive the same validated artifacts. The attachment job uses GitHub Actions' built-in token with `contents: write`; no personal upload token is needed. Repeating the attachment job skips identical assets and refuses to overwrite conflicting checksums.
136
+
137
+ For local validation without publishing: `python scripts/release_assets.py --tag v<package-version> --dist <build-directory>`. The explicit `--upload --repo owner/name` flags perform uploads. Existing releases are not modified by editing this workflow.
138
+
79
139
  ## Three-layer index
80
140
 
81
141
  | Layer | Implementation |
@@ -51,6 +51,16 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
51
51
 
52
52
  `serve` 通过 stdio 提供 JSON-RPC(MCP),暴露七个工具:`codebase_semantic_search`、`codebase_get_symbol_relations`、`codebase_get_optimized_context`、`codebase_get_data_access`、`codebase_impact_analysis`、`codebase_index_status`、`codebase_index_doctor`。常驻的 server 在距离上次检查超过 15 秒时,查询前会重新检查索引(几千个文件无变动约 1 秒)。`codebase_get_data_access` 默认返回摘要(每区 5 条、程序全列),加 `detail` 才列全部;`codebase_semantic_search` 加 `related` 才附依赖文件。
53
53
 
54
+ ## 0.7.0:稳定导航与项目评测
55
+
56
+ 搜索排序不再随页数变化;`pagination.next_cursor` 可用于 `--cursor`/MCP `cursor`,索引或同义词变化时要求重新搜索。`--explain` 显示命中词、扩展来源、排序贡献及 SQL 证据。上下文裁剪优先保留焦点行;同义词自动重载,ONNX 模型和 tokenizer 使用内容指纹。
57
+
58
+ `impact A..B` 和 `A...B` 使用指定版本的快照;多进程索引使用系统锁和一致性读取,繁忙时 CLI 返回退出码 4。`INDEXER_LOCK_TIMEOUT` 与 `INDEXER_BUSY_TIMEOUT` 控制等待秒数,中断批次会在下次索引时恢复。
59
+
60
+ `evaluate --queries .ai_context/eval.json --save .ai_context/report.json` 支持项目问题集、Recall@k、MRR、延迟和估计 token,以及基准比较和质量门槛。数据格式为 `{"version":1,"queries":[{"id":"invoice","query":"读取发票","relevant":[{"file":"src/invoices.py","symbol":"load_invoice"}]}]}`。未达到 `--min-recall`/`--max-p95-ms`/`--max-mean-tokens` 门槛返回退出码 3。`benchmarks/queries_project.json` 是本仓库的人工导航示例,不是独立的真实用户评测。
61
+
62
+ 后续 Release workflow 使用同一份经过验证的构建产物发布到 PyPI 和 GitHub,自动生成 SHA256SUMS;使用 Actions 内置令牌,重复运行跳过相同附件,拒绝覆盖不同内容。修改 workflow 不会变更现有 Release。
63
+
54
64
  ## 诊断、筛选与分页(0.6.0)
55
65
 
56
66
  ```bash
@@ -51,6 +51,29 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
51
51
 
52
52
  `serve` 以 stdio 提供 JSON-RPC(MCP),暴露七個工具:`codebase_semantic_search`、`codebase_get_symbol_relations`、`codebase_get_optimized_context`、`codebase_get_data_access`、`codebase_impact_analysis`、`codebase_index_status`、`codebase_index_doctor`。長駐的 server 在距離上次檢查超過 15 秒時,查詢前會重新檢查索引(幾千個檔案無變動約 1 秒)。`codebase_get_data_access` 預設回摘要(每區 5 筆、程式全列),加 `detail` 才列全部;`codebase_semantic_search` 加 `related` 才附相依檔案。
53
53
 
54
+ ## 0.7.0:穩定導覽與專案評測
55
+
56
+ - 搜尋排序不再隨頁數或每頁筆數改變。JSON 的 `pagination.next_cursor` 可傳給 `--cursor`/MCP `cursor`;查詢與篩選須保持相同,每頁筆數可改變。索引、模型或同義詞設定改變時,游標會要求重新搜尋;舊有 offset 仍可用,但沒有版本保護。
57
+ - `search --explain`/MCP `explain=true` 顯示命中詞、同義詞來源、各層排序貢獻,以及資料存取的 SQL 證據與可信度。分數不是機率,靜態分析也不代表執行時必然經過該路徑。
58
+ - 同義詞設定會在後續查詢重新載入。ONNX 模型、tokenizer、外部權重與設定會納入指紋,內容改變後重建索引;請使用專用模型目錄。大型 context 請求優先保留焦點行,並回報省略/裁切資訊。
59
+ - `impact A..B`/`A...B` 分析指定的兩個 Git 版本,不會混入 HEAD 或未提交的內容;`snapshot=after` 行號屬於區間終點版本。
60
+ - 多程序共用索引時會協調寫入與一致性讀取。忙碌時 CLI 回傳結束碼 4;`INDEXER_LOCK_TIMEOUT`/`INDEXER_BUSY_TIMEOUT` 分別控制鎖定與 SQLite 等待秒數。中斷的批次索引會被偵測並在下次索引時重建;不要刪除使用中的鎖定檔。
61
+
62
+ 將真實使用問題與人工確認的答案放在 `.ai_context/eval.json`:
63
+
64
+ ```json
65
+ {"version":1,"queries":[{"id":"invoice","query":"在哪裡讀取發票?","relevant":[{"file":"src/invoices.py","symbol":"load_invoice"}]}]}
66
+ ```
67
+
68
+ ```bash
69
+ local-codebase-indexer --root <專案> evaluate --queries <專案>/.ai_context/eval.json --save <專案>/.ai_context/baseline.json
70
+ local-codebase-indexer --root <專案> evaluate --queries <專案>/.ai_context/eval.json --baseline <專案>/.ai_context/baseline.json --min-recall 0.8 --max-recall-drop 0.02 --max-p95-ms 500 --max-mean-tokens 1500
71
+ ```
72
+
73
+ 評測追蹤 Recall@k、MRR、命中率、延遲與估計 token;token 使用字元數除以 4 的估算。空題目集或錯誤資料不會算成功,未達門檻回傳結束碼 3。`benchmarks/queries_project.json` 提供本儲存庫的 15 題導覽範例,並非獨立或實際使用者收集的評測;請以自己的真實問題建立品質門檻。
74
+
75
+ 後續 GitHub Release 會自動完成測試、一次建置、版本檢查、PyPI 發布及附件/SHA256SUMS 上傳。附件使用 GitHub Actions 內建權杖,重跑時略過相同檔案,遇到不同校驗值則拒絕覆寫;修改 workflow 本身不會變更現有 Release。
76
+
54
77
  ## 診斷、篩選與分頁(0.6.0)
55
78
 
56
79
  ```bash
@@ -22,6 +22,8 @@ Priorities come from the benchmark (`benchmarks/run_benchmark.py`, baseline in `
22
22
 
23
23
  ## Done
24
24
 
25
+ - 0.7.0: revision-bound search cursors; historical target snapshots; model content identities; process coordination and crash recovery; focus-preserving context; synonym hot reload; retrieval explanations; project evaluation runner; automatic validated Release assets. Representative user-authored evaluation questions and broader semantic retrieval remain ongoing work.
26
+
25
27
  - 0.6.0: complete before/after impact analysis with cached historical indexes; resilient MCP input validation; nested ignore negations; root validation; doctor diagnostics; bounded and filtered search; confidence-filtered data-access pagination with no hidden full-output caps; general paraphrases; mandatory ONNX dependencies in CI. See CHANGELOG.md for validation and limitations.
26
28
 
27
29
  - 0.5.0: a long-lived MCP server re-checks the index (was: once per process); full build 155 s -> 36 s on 2.3k files (C# usings no longer scanned as Python modules, per-file savepoints with batched commits); Chinese comment <-> identifier dictionary mined per project (iWAP paraphrase MRR 0.57 -> 0.64); data-access summary by default (`detail` / `--full`), related files only on request; EF confidence from the context variable's type; NO KNOWN CALLER roots; `impact` tool for a git diff; status shows the running version and warns when the code changed after start
@@ -0,0 +1,21 @@
1
+ {
2
+ "version": 1,
3
+ "description": "Curated navigation smoke questions about this repository's actual implementation. These are not collected user queries or an independent quality benchmark. Replace them with real questions from your own workflow.",
4
+ "queries": [
5
+ {"id":"source-encoding","query":"Which function decodes source bytes using UTF-8, Big5 and cp1252?","language":"python","relevant":[{"file":"local_codebase_indexer/core.py","symbol":"decode_bytes"}]},
6
+ {"id":"extension-language","query":"Map a file extension to its parser language","language":"python","relevant":[{"file":"local_codebase_indexer/scanner.py","symbol":"language_for"}]},
7
+ {"id":"excluded-file","query":"Explain why a path is excluded by ignore rules or its parent directory","language":"python","relevant":[{"file":"local_codebase_indexer/scanner.py","symbol":"explain_path"}]},
8
+ {"id":"mcp-validation","query":"Validate MCP arguments against their schema types and required properties","language":"python","relevant":[{"file":"local_codebase_indexer/mcp_server.py","symbol":"_validate"}]},
9
+ {"id":"writer-coordination","query":"Acquire a process lock with a bounded timeout for an index database","language":"python","relevant":[{"file":"local_codebase_indexer/locking.py","symbol":"index_lock"}]},
10
+ {"id":"rank-fusion","query":"Fuse keyword symbol and vector ranks using reciprocal rank fusion","language":"python","relevant":[{"file":"local_codebase_indexer/query.py","symbol":"semantic_search"}]},
11
+ {"id":"sql-access","query":"Extract SQL reads writes and procedure executions from T-SQL text","language":"python","relevant":[{"file":"local_codebase_indexer/sqlaccess.py","symbol":"extract_sql_accesses"}]},
12
+ {"id":"historical-index","query":"Cache the historical index of a Git revision for impact analysis","language":"python","relevant":[{"file":"local_codebase_indexer/impact.py","symbol":"_historical"}]},
13
+ {"id":"transformer-pooling","query":"Apply mean or CLS pooling and normalize ONNX transformer embeddings","language":"python","relevant":[{"file":"local_codebase_indexer/embedder.py","symbol":"OnnxEmbedder.embed"}]},
14
+ {"id":"gold-validation","query":"Validate evaluation questions and reject empty relevant answers or duplicate IDs","language":"python","relevant":[{"file":"local_codebase_indexer/evaluation.py","symbol":"validate_dataset"}]},
15
+ {"id":"identifier-subwords","query":"Split camelCase and snake_case identifiers into lowercase words","language":"python","relevant":[{"file":"local_codebase_indexer/embedder.py","symbol":"split_identifier"}]},
16
+ {"id":"dictionary-reload","query":"Reload changed project synonym configuration using a content fingerprint","language":"python","relevant":[{"file":"local_codebase_indexer/synonyms.py","symbol":"Expander.reload"}]},
17
+ {"id":"index-location","query":"Choose the default SQLite database path inside the project index directory","language":"python","relevant":[{"file":"local_codebase_indexer/db.py","symbol":"default_db_path"}]},
18
+ {"id":"schema-upgrade","query":"Migrate older SQLite symbol tables by adding missing columns","language":"python","relevant":[{"file":"local_codebase_indexer/db.py","symbol":"_migrate"}]},
19
+ {"id":"release-conflict","query":"Upload missing GitHub release assets while refusing to overwrite different checksums","language":"python","relevant":[{"file":"scripts/release_assets.py","symbol":"upload_assets"}]}
20
+ ]
21
+ }
@@ -1,3 +1,3 @@
1
1
  """Universal Local Codebase Indexer & Context Provider (see index.md)."""
2
2
 
3
- __version__ = "0.6.0"
3
+ __version__ = "0.7.0"
@@ -8,6 +8,7 @@ import os
8
8
  import sys
9
9
 
10
10
  from . import __version__
11
+ from .locking import IndexBusyError
11
12
 
12
13
 
13
14
  def _tools(root: str):
@@ -48,6 +49,8 @@ def build_parser() -> argparse.ArgumentParser:
48
49
  s.add_argument("--query", dest="query_opt")
49
50
  s.add_argument("--limit", type=_positive, default=5)
50
51
  s.add_argument("--offset", type=_nonnegative, default=0)
52
+ s.add_argument("--cursor", help="continue a search against the same index and settings")
53
+ s.add_argument("--explain", action="store_true", help="show matched terms, expansion sources, and ranking evidence")
51
54
  s.add_argument("--file", dest="file_pattern", help="file glob, e.g. '*Repository*.cs'")
52
55
  s.add_argument("--language", help="csharp, python, javascript, typescript, tsx or text")
53
56
  s.add_argument("--related", action="store_true", help="also list dependency-graph neighbours of the top hits")
@@ -79,6 +82,16 @@ def build_parser() -> argparse.ArgumentParser:
79
82
  s.add_argument("base", nargs="?", help="git revision or range (default: uncommitted work against HEAD)")
80
83
  s.add_argument("--full", action="store_true", help="list every changed symbol")
81
84
 
85
+ s = sub.add_parser("evaluate", help="evaluate real project navigation questions and enforce quality budgets")
86
+ s.add_argument("--queries", required=True, help="versioned JSON question set")
87
+ s.add_argument("--limit", type=_positive, default=5)
88
+ s.add_argument("--baseline", help="earlier report for the same question set")
89
+ s.add_argument("--save", help="save a JSON evaluation report")
90
+ s.add_argument("--min-recall", type=float)
91
+ s.add_argument("--max-p95-ms", type=float)
92
+ s.add_argument("--max-mean-tokens", type=float)
93
+ s.add_argument("--max-recall-drop", type=float, default=0.0)
94
+
82
95
  s = sub.add_parser("doctor", help="diagnose index coverage and excluded files")
83
96
  s.add_argument("--path", help="explain why a specific file is excluded")
84
97
  sub.add_parser("stats", help="show index statistics")
@@ -137,7 +150,7 @@ def main(argv: list[str] | None = None) -> int:
137
150
  if not q:
138
151
  print("error: a query is required", file=sys.stderr)
139
152
  return 2
140
- res = t.semantic_search(q, args.limit, args.related, args.file_pattern, args.language, args.offset)
153
+ res = t.semantic_search(q, args.limit, args.related, args.file_pattern, args.language, args.offset, cursor=args.cursor, explain=args.explain)
141
154
  _emit(res, render_search(res), args.json)
142
155
  elif args.cmd in ("relations", "codebase_get_symbol_relations"):
143
156
  sym = args.symbol_opt or args.symbol
@@ -167,12 +180,33 @@ def main(argv: list[str] | None = None) -> int:
167
180
  res = t.index_status()
168
181
  _emit(res, render_status(res), args.json)
169
182
  return 0 if res["state"] in ("fresh", "stale") else 3
183
+ elif args.cmd == "evaluate":
184
+ from .evaluation import evaluate, render_evaluation, save_report
185
+ if args.save and any(os.path.realpath(args.save) == os.path.realpath(path)
186
+ for path in (args.queries, args.baseline) if path):
187
+ raise ValueError('Save the report separately from the input questions and baseline')
188
+ with open(args.queries, encoding='utf-8') as stream:
189
+ dataset = json.load(stream)
190
+ baseline = None
191
+ if args.baseline:
192
+ with open(args.baseline, encoding='utf-8') as stream:
193
+ baseline = json.load(stream)
194
+ res = evaluate(t, dataset, limit=args.limit, baseline=baseline,
195
+ min_recall=args.min_recall, max_p95_ms=args.max_p95_ms,
196
+ max_mean_tokens=args.max_mean_tokens, max_recall_drop=args.max_recall_drop)
197
+ if args.save:
198
+ save_report(res, args.save)
199
+ _emit(res, render_evaluation(res), args.json)
200
+ return 0 if res['passed'] else 3
170
201
  elif args.cmd == "doctor":
171
202
  res = t.doctor(args.path)
172
203
  _emit(res, render_doctor(res), args.json)
173
204
  elif args.cmd == "stats":
174
205
  st = t.stats()
175
206
  _emit(st, "\n".join(f"{k}: {v}" for k, v in st.items()), args.json)
207
+ except IndexBusyError as exc:
208
+ print(f"error: {exc}", file=sys.stderr)
209
+ return 4
176
210
  except (ValueError, OSError) as exc:
177
211
  print(f"error: {exc}", file=sys.stderr)
178
212
  return 1
@@ -9,6 +9,7 @@ import re
9
9
  import sqlite3
10
10
  import subprocess
11
11
  import time
12
+ import uuid
12
13
  from contextlib import contextmanager
13
14
  from collections import defaultdict
14
15
  from dataclasses import dataclass, field
@@ -18,6 +19,7 @@ from .callgraph import resolve_calls
18
19
  from .embedder import get_embedder, split_identifier
19
20
  from .langparse import CLASS_LIKE, parse_file
20
21
  from .scanner import iter_files
22
+ from .locking import IndexBusyError, index_lock, is_busy
21
23
 
22
24
  _TYPE_KINDS = tuple(CLASS_LIKE)
23
25
 
@@ -134,13 +136,33 @@ class Indexer:
134
136
 
135
137
  # ------------------------------------------------------------------ public
136
138
  def update(self, force: bool = False) -> IndexStats:
139
+ with index_lock(self.db_path):
140
+ try:
141
+ if hasattr(self.embedder, 'assert_unchanged'):
142
+ self.embedder.assert_unchanged()
143
+ return self._update(force)
144
+ except sqlite3.OperationalError as exc:
145
+ self.conn.rollback()
146
+ if is_busy(exc):
147
+ raise IndexBusyError('Index database is busy. Retry shortly; another program may be writing to it.') from exc
148
+ raise
149
+ except BaseException:
150
+ self.conn.rollback()
151
+ raise
152
+
153
+ def _update(self, force: bool = False) -> IndexStats:
137
154
  stats = IndexStats()
138
155
  conn = self.conn
156
+ # A crash between batch commits must not leave a partially resolved graph trusted.
157
+ force = force or dbmod.get_meta(conn, 'indexing') == '1'
158
+ dbmod.set_meta(conn, 'indexing', '1')
159
+ conn.commit()
139
160
 
140
161
  if (dbmod.get_meta(conn, "embedder") != self.embedder.name
141
162
  or dbmod.get_meta(conn, "index_version") != dbmod.INDEX_VERSION):
142
163
  # different embedding space or data layout (or first run): rebuild everything
143
164
  dbmod.clear_all(conn)
165
+ dbmod.set_meta(conn, 'indexing', '1')
144
166
  dbmod.set_meta(conn, "embedder", self.embedder.name)
145
167
  dbmod.set_meta(conn, "index_version", dbmod.INDEX_VERSION)
146
168
  conn.commit()
@@ -207,15 +229,18 @@ class Indexer:
207
229
  resolve_page_actions(conn)
208
230
  resolve_ef_access(conn)
209
231
  mine_terms(conn)
210
- self._record_run_meta()
232
+ self._record_run_meta(changed or force)
233
+ conn.execute("DELETE FROM meta WHERE key='indexing'")
211
234
  conn.commit()
212
235
  stats.chunks = conn.execute("SELECT COUNT(*) FROM chunk_metadata").fetchone()[0]
213
236
  stats.symbols = conn.execute("SELECT COUNT(*) FROM symbols").fetchone()[0]
214
237
  return stats
215
238
 
216
239
  # ------------------------------------------------------------------ internals
217
- def _record_run_meta(self) -> None:
240
+ def _record_run_meta(self, changed: bool = False) -> None:
218
241
  conn = self.conn
242
+ if changed or not dbmod.get_meta(conn, 'index_revision'):
243
+ dbmod.set_meta(conn, 'index_revision', uuid.uuid4().hex)
219
244
  dbmod.set_meta(conn, "last_indexed_at", str(int(time.time())))
220
245
  info = git_info(self.root)
221
246
  if info is None:
@@ -4,6 +4,9 @@ from __future__ import annotations
4
4
 
5
5
  import os
6
6
  import sqlite3
7
+ from contextlib import contextmanager
8
+
9
+ from .locking import IndexBusyError, index_lock, is_busy, timeout_seconds
7
10
 
8
11
  INDEX_DIR_NAME = ".ai_context"
9
12
  DB_NAME = "indexer.db"
@@ -166,13 +169,37 @@ def connect(db_path: str) -> sqlite3.Connection:
166
169
  if not os.path.exists(gi):
167
170
  with open(gi, "w", encoding="utf-8") as f:
168
171
  f.write("*\n")
169
- conn = sqlite3.connect(db_path)
170
- conn.row_factory = sqlite3.Row
171
- conn.execute("PRAGMA journal_mode=WAL")
172
- conn.execute("PRAGMA synchronous=NORMAL")
173
- conn.executescript(SCHEMA)
174
- _migrate(conn)
175
- return conn
172
+ with index_lock(db_path):
173
+ conn = sqlite3.connect(db_path, timeout=timeout_seconds('INDEXER_BUSY_TIMEOUT', 5.0))
174
+ conn.row_factory = sqlite3.Row
175
+ try:
176
+ conn.execute("PRAGMA journal_mode=WAL")
177
+ conn.execute("PRAGMA synchronous=NORMAL")
178
+ conn.executescript(SCHEMA)
179
+ _migrate(conn)
180
+ except sqlite3.OperationalError as exc:
181
+ conn.close()
182
+ if is_busy(exc):
183
+ raise IndexBusyError('Index database is busy. Retry shortly; INDEXER_BUSY_TIMEOUT controls the SQLite wait.') from exc
184
+ raise
185
+ return conn
186
+
187
+
188
+ @contextmanager
189
+ def read_snapshot(conn: sqlite3.Connection, db_path: str):
190
+ with index_lock(db_path, shared=True):
191
+ own = not conn.in_transaction
192
+ if own:
193
+ conn.execute('BEGIN')
194
+ try:
195
+ yield
196
+ except BaseException:
197
+ if own:
198
+ conn.rollback()
199
+ raise
200
+ else:
201
+ if own:
202
+ conn.commit()
176
203
 
177
204
 
178
205
  def _migrate(conn: sqlite3.Connection) -> None:
@@ -14,6 +14,8 @@ Two backends:
14
14
  from __future__ import annotations
15
15
 
16
16
  import math
17
+ import hashlib
18
+ import json
17
19
  import os
18
20
  import re
19
21
  import zlib
@@ -76,6 +78,23 @@ class HashingEmbedder:
76
78
  return np.stack([self._vec(t) for t in texts]) if texts else np.zeros((0, DIM), np.float32)
77
79
 
78
80
 
81
+ def _model_files(model_dir: str):
82
+ for directory, dirs, files in os.walk(model_dir):
83
+ dirs[:] = sorted(d for d in dirs if d not in ('.git', '.cache', '__pycache__', '.ai_context', '.venv', 'venv'))
84
+ for filename in sorted(files):
85
+ path = os.path.join(directory, filename)
86
+ if os.path.isfile(path):
87
+ yield path
88
+
89
+
90
+ def _model_state(model_dir: str) -> tuple:
91
+ state = []
92
+ for path in _model_files(model_dir):
93
+ stat = os.stat(path)
94
+ state.append((os.path.relpath(path, model_dir), stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns, stat.st_ino))
95
+ return tuple(state)
96
+
97
+
79
98
  class OnnxEmbedder:
80
99
  dim = DIM
81
100
 
@@ -83,7 +102,8 @@ class OnnxEmbedder:
83
102
  import onnxruntime as ort
84
103
  from tokenizers import Tokenizer
85
104
 
86
- self.name = "onnx:" + os.path.basename(os.path.abspath(model_dir))
105
+ self.model_dir = os.path.abspath(model_dir)
106
+ self._state = _model_state(self.model_dir)
87
107
  self.tok = Tokenizer.from_file(os.path.join(model_dir, "tokenizer.json"))
88
108
  self.tok.enable_truncation(max_length=256)
89
109
  self.tok.enable_padding()
@@ -96,15 +116,38 @@ class OnnxEmbedder:
96
116
  if self.pooling not in ("mean", "cls"):
97
117
  raise ValueError("INDEXER_ONNX_POOLING must be 'mean' or 'cls'")
98
118
  self.query_prefix = os.environ.get("INDEXER_ONNX_QUERY_PREFIX", "")
99
- self.name += f":{self.pooling}"
100
119
  out_dim = self.session.get_outputs()[0].shape[-1]
101
120
  if isinstance(out_dim, int):
102
121
  self.dim = out_dim
122
+ # Include tokenizer and external weight files, not just the directory name.
123
+ digest = hashlib.sha256()
124
+ for path in _model_files(self.model_dir):
125
+ relative = os.path.relpath(path, self.model_dir).replace(os.sep, '/')
126
+ digest.update(relative.encode('utf-8') + b'\0')
127
+ content = hashlib.sha256()
128
+ with open(path, 'rb') as stream:
129
+ for block in iter(lambda: stream.read(1024 * 1024), b''):
130
+ content.update(block)
131
+ digest.update(content.digest())
132
+ digest.update(json.dumps({'pooling': self.pooling, 'dim': self.dim,
133
+ 'query_prefix': self.query_prefix}, sort_keys=True).encode())
134
+ self.fingerprint = digest.hexdigest()
135
+ self.name = f"onnx:{os.path.basename(os.path.abspath(model_dir))}:{self.fingerprint}:{self.pooling}"
136
+ self.assert_unchanged()
137
+
138
+ def assert_unchanged(self) -> None:
139
+ try:
140
+ unchanged = _model_state(self.model_dir) == self._state
141
+ except OSError:
142
+ unchanged = False
143
+ if not unchanged:
144
+ raise ValueError('ONNX model files changed while loaded; restart the server to load the new model and rebuild its index.')
103
145
 
104
146
  def embed_query(self, text: str) -> np.ndarray:
105
147
  return self.embed([self.query_prefix + text])[0]
106
148
 
107
149
  def embed(self, texts: Sequence[str]) -> np.ndarray:
150
+ self.assert_unchanged()
108
151
  if not texts:
109
152
  return np.zeros((0, self.dim), np.float32)
110
153
  out = []