local-codebase-indexer 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {local_codebase_indexer-0.3.2/local_codebase_indexer.egg-info → local_codebase_indexer-0.4.0}/PKG-INFO +7 -5
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/README.md +6 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/README.zh-CN.md +6 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/README.zh-TW.md +6 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/TODO.md +5 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/__init__.py +1 -1
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/cli.py +4 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/core.py +90 -1
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/cscalls.py +79 -3
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/csdata.py +124 -5
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/dataflow.py +70 -5
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/db.py +1 -1
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/langparse.py +24 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/mcp_server.py +4 -4
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/query.py +48 -11
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/scanner.py +40 -3
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/sqlaccess.py +1 -1
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0/local_codebase_indexer.egg-info}/PKG-INFO +7 -5
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/server.json +2 -2
- local_codebase_indexer-0.4.0/tests/test_dataaccess.py +345 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/tests/test_indexer.py +30 -1
- local_codebase_indexer-0.3.2/tests/test_dataaccess.py +0 -144
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/AGENTS.md +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/CLAUDE.md +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/MANIFEST.in +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/README.md +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/baseline.json +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Auth/LoginService.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Controllers/LotController.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Controllers/ScrapController.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Data/MesDbContext.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Legacy/frmChangeLine.frm +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Models/DrillBreakEvent.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Models/Lot.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Models/ScrapRecord.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Printing/LabelPrinter.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Repositories/LotRepository.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Repositories/ScrapRepository.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Services/IScrapService.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Services/InspectionService.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Services/LotService.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Services/ScrapService.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Sql/MES_Lot.sql +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Sql/usp_Inspection_Report.sql +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Sql/usp_Lot_UpdateStatus.sql +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Sql/usp_Scrap_Insert.sql +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Utils/DateRangeHelper.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Utils/PartNumberDecoder.cs +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Web/scrapForm.ts +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/Web.config +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/corpus/tools/export_report.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/queries_stdlib.json +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/queries_synthetic.json +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/benchmarks/run_benchmark.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/index.md +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/llms.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/__main__.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/callgraph.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/embedder.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/synonyms.py +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer.egg-info/SOURCES.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer.egg-info/dependency_links.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer.egg-info/entry_points.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer.egg-info/requires.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer.egg-info/top_level.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/pyproject.toml +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/requirements.txt +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/setup.cfg +0 -0
- {local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/tests/test_onnx_plumbing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: local-codebase-indexer
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Local codebase indexer and context provider for AI coding assistants: MCP server and CLI with SQL data-access and C# call relations. No cloud, no server.
|
|
5
5
|
Author: sfy257000
|
|
6
6
|
Project-URL: Homepage, https://github.com/sfy257000/local-codebase-indexer
|
|
@@ -88,18 +88,20 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
|
|
|
88
88
|
| Topology | `using` / `import` resolved to file dependencies, inheritance (base lists) and type usage (`invocation`), stored in the `dependencies` table |
|
|
89
89
|
| Semantic | 384-dim chunk embeddings in `vss_chunks`; queries fuse FTS5 keyword, symbol-name and vector rankings with Reciprocal Rank Fusion, then expand to the dependency neighbours of the top 3 files |
|
|
90
90
|
|
|
91
|
-
Encodings: UTF-8 first, then Big5, then cp1252. Files over 5MB are skipped. Root and nested `.gitignore` files are respected, and `.git`, `bin`, `obj`, `node_modules` and similar directories are always skipped.
|
|
91
|
+
Encodings: UTF-8 first, then Big5, then cp1252. Files over 5MB are skipped. Root and nested `.gitignore` files are respected, and `.git`, `bin`, `obj`, `node_modules` and similar directories are always skipped. `*.min.js` / `*.min.css` are skipped too. Third-party or generated code that is committed to the repo can be excluded with `.ai_context/ignore` (gitignore syntax), e.g. `Scripts/*`, `!Scripts/Custom/`, `Web References/` (keep generated EF database-first models: their DbContext declares the DbSets).
|
|
92
92
|
|
|
93
93
|
## Differences from the spec
|
|
94
94
|
|
|
95
95
|
- **Vector storage**: `sqlite-vss` needs a native extension and is no longer maintained, so `vss_chunks` stores float32 BLOBs and cosine similarity is computed with numpy. No extension to install.
|
|
96
|
-
- **Embedding model**: the default is a built-in hashing embedder (identifier sub-words, bigrams, CJK character bigrams). It is fast and reproducible but **does not understand synonyms**. For real transformer embeddings, export `all-MiniLM-L6-v2` or `bge-small-en-v1.5` to ONNX, install `onnxruntime` and `tokenizers`, and set `INDEXER_ONNX_MODEL=<folder with model.onnx and tokenizer.json>` (
|
|
97
|
-
- **Call relations** for C# are resolved with limited type inference (using / namespaces, field and constructor-injected types, locals, interface to implementation), and every edge carries a confidence: `confirmed` (the receiver type is known), `probable` (reached through an interface or a single candidate) or `candidate` (name match only). There is no Roslyn: no overload resolution by argument type, no delegates / lambdas / extension methods, and call chains are followed one level. Inheritance edges come from the AST and are reliable.
|
|
96
|
+
- **Embedding model**: the default is a built-in hashing embedder (identifier sub-words, bigrams, CJK character bigrams). It is fast and reproducible but **does not understand synonyms**. For real transformer embeddings, export `all-MiniLM-L6-v2` or `bge-small-en-v1.5` to ONNX, install `onnxruntime` and `tokenizers`, and set `INDEXER_ONNX_MODEL=<folder with model.onnx and tokenizer.json>` (mean pooling suits MiniLM, bge models recommend CLS pooling). Switching embedders triggers a full re-index. Measured (0.4.0): `all-MiniLM-L6-v2` raises benchmark MRR by 0.08-0.13, but on a real 30k-chunk C# / Chinese-comment codebase it scored lower than hashing on paraphrased queries (MRR 0.29 vs 0.48; multilingual MiniLM-L12: 0.41) while a full index took 35-43 minutes instead of 2. Hashing stays the default; for domain jargon, `.ai_context/synonyms.json` helps more than a general model.
|
|
97
|
+
- **Call relations** for C# are resolved with limited type inference (using / namespaces, field and constructor-injected types, `Lazy<T>.Value`, locals, `foreach (var x in items.Where(..))` element types, interface to implementation), and every edge carries a confidence: `confirmed` (the receiver type is known), `probable` (reached through an interface or a single candidate) or `candidate` (name match only). There is no Roslyn: no overload resolution by argument type, no delegates / lambdas / extension methods, and call chains are followed one level. Inheritance edges come from the AST and are reliable.
|
|
98
98
|
- Extra schema additions: `symbols.qualified_name`, `chunk_metadata.symbol_id`, and the `raw_refs`, `chunks_fts` and `meta` tables.
|
|
99
99
|
|
|
100
100
|
## Data access (SQL)
|
|
101
101
|
|
|
102
|
-
`data-access <table-or-procedure>` answers "what writes / reads / executes X". It is built from T-SQL (INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC, with alias resolution; CTEs, temp tables and system objects excluded), from C# SQL strings (literals, `+`, interpolation, `string.Format`, consts, locals) passed to Dapper, `SqlCommand` or EF `FromSql`, from stored-procedure names used with `CommandType.StoredProcedure`, and from EF `ToTable` / `[Table]` mappings. A procedure's callers are indirect writers of the tables it writes (followed up to 4 hops). Each entry has a confidence; **dynamic SQL (`EXEC(@sql)`, `sp_executesql`, SQL built from unknown values) cannot be resolved and is listed separately as `uncertain`**, never mixed into the confirmed lists. Known gaps: `FROM a JOIN b ON ..., c` loses `c`, and a stored-procedure name passed to a non-database method is not seen.
|
|
102
|
+
`data-access <table-or-procedure>` answers "what writes / reads / executes X". It is built from T-SQL (INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC, with alias resolution; CTEs, temp tables and system objects excluded), from C# SQL strings (literals, `+`, interpolation, `string.Format`, consts, locals, `StringBuilder.Append` / `AppendLine` / `AppendFormat`) passed to Dapper, `SqlCommand` or EF `FromSql`, from stored-procedure names used with `CommandType.StoredProcedure`, from EF LINQ on `DbSet<T>` members (`ctx.Orders.Where(..)` / `.Add(..)` / `.Remove(..)`, `from o in ctx.Orders`; `probable`, since only the member name is matched) and from EF `ToTable` / `[Table]` mappings. A procedure's callers are indirect writers of the tables it writes (followed up to 4 hops). `programs` (USED BY PROGRAMS) follows confirmed / probable C# calls upward from those accessors to `*Controller` methods (up to 8 hops) (including from an implementation to the callers of the interface method it implements, for `foreach (var step in steps) step.Run()` pipelines) and groups them per controller file, with the Razor pages (`.cshtml`) whose `Url.Action("Action"[, "Controller"])` or AngularJS `ControllerPath + 'Action'` (controller from `$scope.Controller = 'X'`) targets those actions, so a table maps to the screens that touch it. A method that reads a DbSet and calls `SaveChanges` without an explicit `Add` / `Remove` may update tracked entities; it is listed as a possible (`uncertain`) writer. Each entry has a confidence; **dynamic SQL (`EXEC(@sql)`, `sp_executesql`, SQL built from unknown values) cannot be resolved and is listed separately as `uncertain`**, never mixed into the confirmed lists. Known gaps: `FROM a JOIN b ON ..., c` loses `c`, and a stored-procedure name passed to a non-database method is not seen.
|
|
103
|
+
|
|
104
|
+
Stored procedures that live only in the database can be exported to a folder outside the repo and listed in `.ai_context/extra_roots` (one folder per line, absolute or relative to the root); their files are indexed under their absolute paths.
|
|
103
105
|
|
|
104
106
|
Natural-language search routes questions such as "who modifies MES_ScrapRecord" or "哪些程式會寫入 MES_Lot" to this index first.
|
|
105
107
|
|
|
@@ -53,18 +53,20 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
|
|
|
53
53
|
| Topology | `using` / `import` resolved to file dependencies, inheritance (base lists) and type usage (`invocation`), stored in the `dependencies` table |
|
|
54
54
|
| Semantic | 384-dim chunk embeddings in `vss_chunks`; queries fuse FTS5 keyword, symbol-name and vector rankings with Reciprocal Rank Fusion, then expand to the dependency neighbours of the top 3 files |
|
|
55
55
|
|
|
56
|
-
Encodings: UTF-8 first, then Big5, then cp1252. Files over 5MB are skipped. Root and nested `.gitignore` files are respected, and `.git`, `bin`, `obj`, `node_modules` and similar directories are always skipped.
|
|
56
|
+
Encodings: UTF-8 first, then Big5, then cp1252. Files over 5MB are skipped. Root and nested `.gitignore` files are respected, and `.git`, `bin`, `obj`, `node_modules` and similar directories are always skipped. `*.min.js` / `*.min.css` are skipped too. Third-party or generated code that is committed to the repo can be excluded with `.ai_context/ignore` (gitignore syntax), e.g. `Scripts/*`, `!Scripts/Custom/`, `Web References/` (keep generated EF database-first models: their DbContext declares the DbSets).
|
|
57
57
|
|
|
58
58
|
## Differences from the spec
|
|
59
59
|
|
|
60
60
|
- **Vector storage**: `sqlite-vss` needs a native extension and is no longer maintained, so `vss_chunks` stores float32 BLOBs and cosine similarity is computed with numpy. No extension to install.
|
|
61
|
-
- **Embedding model**: the default is a built-in hashing embedder (identifier sub-words, bigrams, CJK character bigrams). It is fast and reproducible but **does not understand synonyms**. For real transformer embeddings, export `all-MiniLM-L6-v2` or `bge-small-en-v1.5` to ONNX, install `onnxruntime` and `tokenizers`, and set `INDEXER_ONNX_MODEL=<folder with model.onnx and tokenizer.json>` (
|
|
62
|
-
- **Call relations** for C# are resolved with limited type inference (using / namespaces, field and constructor-injected types, locals, interface to implementation), and every edge carries a confidence: `confirmed` (the receiver type is known), `probable` (reached through an interface or a single candidate) or `candidate` (name match only). There is no Roslyn: no overload resolution by argument type, no delegates / lambdas / extension methods, and call chains are followed one level. Inheritance edges come from the AST and are reliable.
|
|
61
|
+
- **Embedding model**: the default is a built-in hashing embedder (identifier sub-words, bigrams, CJK character bigrams). It is fast and reproducible but **does not understand synonyms**. For real transformer embeddings, export `all-MiniLM-L6-v2` or `bge-small-en-v1.5` to ONNX, install `onnxruntime` and `tokenizers`, and set `INDEXER_ONNX_MODEL=<folder with model.onnx and tokenizer.json>` (mean pooling suits MiniLM, bge models recommend CLS pooling). Switching embedders triggers a full re-index. Measured (0.4.0): `all-MiniLM-L6-v2` raises benchmark MRR by 0.08-0.13, but on a real 30k-chunk C# / Chinese-comment codebase it scored lower than hashing on paraphrased queries (MRR 0.29 vs 0.48; multilingual MiniLM-L12: 0.41) while a full index took 35-43 minutes instead of 2. Hashing stays the default; for domain jargon, `.ai_context/synonyms.json` helps more than a general model.
|
|
62
|
+
- **Call relations** for C# are resolved with limited type inference (using / namespaces, field and constructor-injected types, `Lazy<T>.Value`, locals, `foreach (var x in items.Where(..))` element types, interface to implementation), and every edge carries a confidence: `confirmed` (the receiver type is known), `probable` (reached through an interface or a single candidate) or `candidate` (name match only). There is no Roslyn: no overload resolution by argument type, no delegates / lambdas / extension methods, and call chains are followed one level. Inheritance edges come from the AST and are reliable.
|
|
63
63
|
- Extra schema additions: `symbols.qualified_name`, `chunk_metadata.symbol_id`, and the `raw_refs`, `chunks_fts` and `meta` tables.
|
|
64
64
|
|
|
65
65
|
## Data access (SQL)
|
|
66
66
|
|
|
67
|
-
`data-access <table-or-procedure>` answers "what writes / reads / executes X". It is built from T-SQL (INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC, with alias resolution; CTEs, temp tables and system objects excluded), from C# SQL strings (literals, `+`, interpolation, `string.Format`, consts, locals) passed to Dapper, `SqlCommand` or EF `FromSql`, from stored-procedure names used with `CommandType.StoredProcedure`, and from EF `ToTable` / `[Table]` mappings. A procedure's callers are indirect writers of the tables it writes (followed up to 4 hops). Each entry has a confidence; **dynamic SQL (`EXEC(@sql)`, `sp_executesql`, SQL built from unknown values) cannot be resolved and is listed separately as `uncertain`**, never mixed into the confirmed lists. Known gaps: `FROM a JOIN b ON ..., c` loses `c`, and a stored-procedure name passed to a non-database method is not seen.
|
|
67
|
+
`data-access <table-or-procedure>` answers "what writes / reads / executes X". It is built from T-SQL (INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC, with alias resolution; CTEs, temp tables and system objects excluded), from C# SQL strings (literals, `+`, interpolation, `string.Format`, consts, locals, `StringBuilder.Append` / `AppendLine` / `AppendFormat`) passed to Dapper, `SqlCommand` or EF `FromSql`, from stored-procedure names used with `CommandType.StoredProcedure`, from EF LINQ on `DbSet<T>` members (`ctx.Orders.Where(..)` / `.Add(..)` / `.Remove(..)`, `from o in ctx.Orders`; `probable`, since only the member name is matched) and from EF `ToTable` / `[Table]` mappings. A procedure's callers are indirect writers of the tables it writes (followed up to 4 hops). `programs` (USED BY PROGRAMS) follows confirmed / probable C# calls upward from those accessors to `*Controller` methods (up to 8 hops) (including from an implementation to the callers of the interface method it implements, for `foreach (var step in steps) step.Run()` pipelines) and groups them per controller file, with the Razor pages (`.cshtml`) whose `Url.Action("Action"[, "Controller"])` or AngularJS `ControllerPath + 'Action'` (controller from `$scope.Controller = 'X'`) targets those actions, so a table maps to the screens that touch it. A method that reads a DbSet and calls `SaveChanges` without an explicit `Add` / `Remove` may update tracked entities; it is listed as a possible (`uncertain`) writer. Each entry has a confidence; **dynamic SQL (`EXEC(@sql)`, `sp_executesql`, SQL built from unknown values) cannot be resolved and is listed separately as `uncertain`**, never mixed into the confirmed lists. Known gaps: `FROM a JOIN b ON ..., c` loses `c`, and a stored-procedure name passed to a non-database method is not seen.
|
|
68
|
+
|
|
69
|
+
Stored procedures that live only in the database can be exported to a folder outside the repo and listed in `.ai_context/extra_roots` (one folder per line, absolute or relative to the root); their files are indexed under their absolute paths.
|
|
68
70
|
|
|
69
71
|
Natural-language search routes questions such as "who modifies MES_ScrapRecord" or "哪些程式會寫入 MES_Lot" to this index first.
|
|
70
72
|
|
|
@@ -49,18 +49,20 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
|
|
|
49
49
|
| Topology | `using` / `import` 解析为文件依赖;继承(base list);类型使用(`invocation`)。存放在 `dependencies` 表 |
|
|
50
50
|
| Semantic | 代码块嵌入 384 维,存入 `vss_chunks`;查询时用 FTS5 关键字、符号名称、向量三路做 Reciprocal Rank Fusion,再展开前 3 名文件的依赖邻居 |
|
|
51
51
|
|
|
52
|
-
编码:先尝试 UTF-8,再尝试 Big5、cp1252。单个文件超过 5MB 不索引;遵守根目录与子目录的 `.gitignore`;固定跳过 `.git`、`bin`、`obj`、`node_modules`
|
|
52
|
+
编码:先尝试 UTF-8,再尝试 Big5、cp1252。单个文件超过 5MB 不索引;遵守根目录与子目录的 `.gitignore`;固定跳过 `.git`、`bin`、`obj`、`node_modules` 等。`*.min.js` / `*.min.css` 也跳过。已提交进仓库的第三方或生成的代码可用 `.ai_context/ignore`(gitignore 语法)排除,例如 `Scripts/*`、`!Scripts/Custom/`、`Web References/`(EF database-first 生成的 model 不要排除,DbSet 声明在它的 DbContext 里)。
|
|
53
53
|
|
|
54
54
|
## 与规格的差异
|
|
55
55
|
|
|
56
56
|
- **向量存储**:规格中的 `sqlite-vss` 需要原生扩展且已停止维护,这里 `vss_chunks` 改为存 float32 BLOB,用 numpy 暴力计算 cosine,不需要额外安装扩展。
|
|
57
|
-
- **嵌入模型**:默认是内置的 hashing 嵌入(标识符拆词、双词组、中文双字),快速、可复现,但**不理解同义词**。要使用真正的 transformer,把 `all-MiniLM-L6-v2` 或 `bge-small-en-v1.5` 导出为 ONNX,安装 `onnxruntime`、`tokenizers`,并设置 `INDEXER_ONNX_MODEL=<包含 model.onnx 与 tokenizer.json
|
|
58
|
-
- **C# 调用关系**以有限度的类型推断解析(using / namespace
|
|
57
|
+
- **嵌入模型**:默认是内置的 hashing 嵌入(标识符拆词、双词组、中文双字),快速、可复现,但**不理解同义词**。要使用真正的 transformer,把 `all-MiniLM-L6-v2` 或 `bge-small-en-v1.5` 导出为 ONNX,安装 `onnxruntime`、`tokenizers`,并设置 `INDEXER_ONNX_MODEL=<包含 model.onnx 与 tokenizer.json 的文件夹>`(目前使用平均池化,适合 MiniLM,bge 系列建议使用 CLS 池化)。切换嵌入后会自动重建索引。实测(0.4.0):`all-MiniLM-L6-v2` 让 benchmark MRR 提高 0.08–0.13,但在真实的 3 万 chunk C# / 中文注释项目上,换个说法的查询反而比 hashing 差(MRR 0.29 对 0.48;多语 MiniLM-L12 为 0.41),完整建索引从 2 分钟变成 35–43 分钟。因此默认维持 hashing;领域术语用 `.ai_context/synonyms.json` 比通用模型有效。
|
|
58
|
+
- **C# 调用关系**以有限度的类型推断解析(using / namespace、字段与构造函数注入的类型、`Lazy<T>.Value`、局部变量、`foreach (var x in items.Where(..))` 的元素类型、接口对应实现),每条关系都标置信等级:`confirmed`(接收者类型已知)、`probable`(经由接口或唯一候选)、`candidate`(仅名称相符)。没有 Roslyn,所以不做按参数类型的重载解析、不处理委托 / lambda / 扩展方法,调用链只追一层。继承关系来自 AST,可靠。
|
|
59
59
|
- 额外增加了 `symbols.qualified_name`、`chunk_metadata.symbol_id`、`raw_refs`、`chunks_fts` 与 `meta` 表。
|
|
60
60
|
|
|
61
61
|
## 数据访问(SQL)
|
|
62
62
|
|
|
63
|
-
`data-access <数据表或存储过程>` 回答“谁写入 / 读取 / 执行 X”。数据来源:T-SQL(INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC,含别名解析,排除 CTE、临时表、系统对象)、C# 的 SQL 字符串(字面值、`+`、字符串插值、`string.Format
|
|
63
|
+
`data-access <数据表或存储过程>` 回答“谁写入 / 读取 / 执行 X”。数据来源:T-SQL(INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC,含别名解析,排除 CTE、临时表、系统对象)、C# 的 SQL 字符串(字面值、`+`、字符串插值、`string.Format`、常量、局部变量、`StringBuilder.Append` / `AppendLine` / `AppendFormat`)经 Dapper、`SqlCommand` 或 EF `FromSql` 发出、配合 `CommandType.StoredProcedure` 的存储过程名称、对 `DbSet<T>` 成员的 EF LINQ(`ctx.Orders.Where(..)` / `.Add(..)` / `.Remove(..)`、`from o in ctx.Orders`;只比对成员名称,所以是 `probable`),以及 EF 的 `ToTable` / `[Table]` 映射。调用某存储过程的代码视为该过程所写数据表的间接写入者(最多追 4 层)。`programs`(USED BY PROGRAMS)沿 confirmed / probable 的 C# 调用往上追到 `*Controller` 方法(最多 8 层),(实现方法也会接到它所实现接口方法的调用者,涵盖 `foreach (var step in steps) step.Run()` 这种管线),按 controller 文件分组,并列出 `Url.Action("Action"[, "Controller"])` 或 AngularJS `ControllerPath + 'Action'`(controller 取自 `$scope.Controller = 'X'`)指向这些 action 的 Razor 页面(`.cshtml`),查一张表就知道哪些画面会动到它。方法读了某 DbSet 又调用 `SaveChanges`、但没有明确 `Add` / `Remove` 时,可能是在更新跟踪中的实体,列为可能(`uncertain`)写入者。每条都有置信等级;**动态 SQL(`EXEC(@sql)`、`sp_executesql`、由未知值组成的 SQL)无法解析,会单独列为 `uncertain`**,不会混进确定的清单。已知缺口:`FROM a JOIN b ON ..., c` 会漏掉 `c`;把存储过程名称传给非数据库方法时看不到。
|
|
64
|
+
|
|
65
|
+
只存在数据库里的存储过程,可导出到仓库外的文件夹,再写进 `.ai_context/extra_roots`(一行一个文件夹,绝对路径或相对于项目根目录);这些文件以绝对路径索引。
|
|
64
66
|
|
|
65
67
|
自然语言搜索遇到“谁修改 MES_ScrapRecord”“哪些程序会写入 MES_Lot”这类问题,会先走这个索引。
|
|
66
68
|
|
|
@@ -49,18 +49,20 @@ claude mcp add local-indexer -- local-codebase-indexer --root /path/to/project s
|
|
|
49
49
|
| Topology | `using` / `import` 解析成檔案相依;繼承(base list);型別使用(`invocation`)。存在 `dependencies` 表 |
|
|
50
50
|
| Semantic | 區塊嵌入 384 維,存 `vss_chunks`;查詢時以 FTS5 關鍵字、符號名稱、向量三路做 Reciprocal Rank Fusion,再展開前 3 名檔案的相依鄰居 |
|
|
51
51
|
|
|
52
|
-
編碼:先試 UTF-8,再試 Big5、cp1252。單檔超過 5MB 不索引;遵守根目錄與子目錄的 `.gitignore`;固定略過 `.git`、`bin`、`obj`、`node_modules`
|
|
52
|
+
編碼:先試 UTF-8,再試 Big5、cp1252。單檔超過 5MB 不索引;遵守根目錄與子目錄的 `.gitignore`;固定略過 `.git`、`bin`、`obj`、`node_modules` 等。`*.min.js` / `*.min.css` 也略過。已 commit 進 repo 的第三方或產生的程式可用 `.ai_context/ignore`(gitignore 語法)排除,例如 `Scripts/*`、`!Scripts/Custom/`、`Web References/`(EF database-first 產生的 model 不要排除,DbSet 宣告在它的 DbContext 裡)。
|
|
53
53
|
|
|
54
54
|
## 與規格的差異
|
|
55
55
|
|
|
56
56
|
- **向量儲存**:規格的 `sqlite-vss` 需要原生擴充且已停止維護,這裡 `vss_chunks` 改存 float32 BLOB,用 numpy 暴力計算 cosine,不需要額外安裝擴充。
|
|
57
|
-
- **嵌入模型**:預設是內建的 hashing 嵌入(識別字拆詞、雙詞組、中文雙字),快速、可重現,但**不理解同義詞**。要用真正的 transformer,匯出 `all-MiniLM-L6-v2` 或 `bge-small-en-v1.5` 為 ONNX,裝 `onnxruntime`、`tokenizers`,並設 `INDEXER_ONNX_MODEL=<含 model.onnx 與 tokenizer.json
|
|
58
|
-
- **C# 呼叫關係**以有限度的型別推斷解析(using / namespace
|
|
57
|
+
- **嵌入模型**:預設是內建的 hashing 嵌入(識別字拆詞、雙詞組、中文雙字),快速、可重現,但**不理解同義詞**。要用真正的 transformer,匯出 `all-MiniLM-L6-v2` 或 `bge-small-en-v1.5` 為 ONNX,裝 `onnxruntime`、`tokenizers`,並設 `INDEXER_ONNX_MODEL=<含 model.onnx 與 tokenizer.json 的資料夾>`(目前用平均池化,適合 MiniLM,bge 系列建議用 CLS 池化)。切換嵌入後會自動重建索引。實測(0.4.0):`all-MiniLM-L6-v2` 讓 benchmark MRR 提高 0.08–0.13,但在真實的 3 萬 chunk C# / 中文註解專案上,換句話說的查詢反而比 hashing 差(MRR 0.29 對 0.48;多語 MiniLM-L12 為 0.41),完整建索引從 2 分鐘變成 35–43 分鐘。因此預設維持 hashing;領域行話用 `.ai_context/synonyms.json` 比通用模型有效。
|
|
58
|
+
- **C# 呼叫關係**以有限度的型別推斷解析(using / namespace、欄位與建構式注入的型別、`Lazy<T>.Value`、區域變數、`foreach (var x in items.Where(..))` 的元素型別、介面對應實作),每條關係都標信心等級:`confirmed`(接收者型別已知)、`probable`(經由介面或唯一候選)、`candidate`(僅名稱相符)。沒有 Roslyn,所以不做依引數型別的多載解析、不處理委派 / lambda / 擴充方法,呼叫鏈只追一層。繼承關係來自 AST,可靠。
|
|
59
59
|
- 額外加了 `symbols.qualified_name`、`chunk_metadata.symbol_id`、`raw_refs`、`chunks_fts` 與 `meta` 表。
|
|
60
60
|
|
|
61
61
|
## 資料存取(SQL)
|
|
62
62
|
|
|
63
|
-
`data-access <資料表或預存程序>` 回答「誰寫入 / 讀取 / 執行 X」。資料來源:T-SQL(INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC,含別名解析,排除 CTE、暫存表、系統物件)、C# 的 SQL 字串(字面值、`+`、字串內插、`string.Format
|
|
63
|
+
`data-access <資料表或預存程序>` 回答「誰寫入 / 讀取 / 執行 X」。資料來源:T-SQL(INSERT / UPDATE / DELETE / MERGE / TRUNCATE / SELECT INTO / FROM / JOIN / EXEC,含別名解析,排除 CTE、暫存表、系統物件)、C# 的 SQL 字串(字面值、`+`、字串內插、`string.Format`、常數、區域變數、`StringBuilder.Append` / `AppendLine` / `AppendFormat`)經 Dapper、`SqlCommand` 或 EF `FromSql` 送出、搭配 `CommandType.StoredProcedure` 的 SP 名稱、對 `DbSet<T>` 成員的 EF LINQ(`ctx.Orders.Where(..)` / `.Add(..)` / `.Remove(..)`、`from o in ctx.Orders`;只比對成員名稱,所以是 `probable`),以及 EF 的 `ToTable` / `[Table]` 對應。呼叫某 SP 的程式視為該 SP 所寫資料表的間接寫入者(最多追 4 層)。`programs`(USED BY PROGRAMS)沿 confirmed / probable 的 C# 呼叫往上追到 `*Controller` 方法(最多 8 層),(實作方法也會接到它所實作介面方法的呼叫者,涵蓋 `foreach (var step in steps) step.Run()` 這種管線),依 controller 檔案分組,並列出 `Url.Action("Action"[, "Controller"])` 或 AngularJS `ControllerPath + 'Action'`(controller 取自 `$scope.Controller = 'X'`)指向這些 action 的 Razor 頁面(`.cshtml`),查一張表就知道哪些畫面會動到它。方法讀了某 DbSet 又呼叫 `SaveChanges`、但沒有明確 `Add` / `Remove` 時,可能是在更新追蹤中的實體,列為可能(`uncertain`)寫入者。每筆都有信心等級;**動態 SQL(`EXEC(@sql)`、`sp_executesql`、由未知值組成的 SQL)無法解析,會獨立列為 `uncertain`**,不會混進確定的清單。已知缺口:`FROM a JOIN b ON ..., c` 會漏掉 `c`;把 SP 名稱傳給非資料庫方法時看不到。
|
|
64
|
+
|
|
65
|
+
只存在資料庫裡的預存程序,可匯出到 repo 外的資料夾,再寫進 `.ai_context/extra_roots`(一行一個資料夾,絕對路徑或相對於專案根目錄);這些檔案以絕對路徑索引。
|
|
64
66
|
|
|
65
67
|
自然語言搜尋遇到「誰修改 MES_ScrapRecord」「哪些程式會寫入 MES_Lot」這類問題,會先走這個索引。
|
|
66
68
|
|
|
@@ -5,15 +5,14 @@ Priorities come from the benchmark (`benchmarks/run_benchmark.py`, baseline in `
|
|
|
5
5
|
## Next
|
|
6
6
|
|
|
7
7
|
0. **Publish to the MCP registry** (not done yet). PyPI 0.3.0 is out and carries the `mcp-name` marker; `server.json` is ready. Run `mcp-publisher login github` then `mcp-publisher publish` from the repo folder, then verify with `curl "https://registry.modelcontextprotocol.io/v0.1/servers?search=local-codebase-indexer"`. Repeat after each release (bump both versions in `server.json`).
|
|
8
|
-
1. **
|
|
9
|
-
|
|
10
|
-
2. **Resolve `FROM a JOIN b ON ..., c`** (a comma list after a JOIN loses `c`), non-sink SQL proc-name literals (`Run("usp_X")`), and EF query tracing (only the table mapping is recorded now).
|
|
11
|
-
3. **C# call graph gaps**: overload resolution by argument type, delegates / lambdas / extension methods, call chains longer than one level. Without Roslyn these stay approximate; edges carry `confirmed` / `probable` / `candidate`.
|
|
8
|
+
1. **Resolve `FROM a JOIN b ON ..., c`** (a comma list after a JOIN loses `c`), non-sink SQL proc-name literals (`Run("usp_X")`), EF `this.ToTable(..)` inside `EntityTypeConfiguration<T>` (entity not taken from the base class yet), and EF updates through tracked entities when the entity was loaded in another method than the one calling `SaveChanges` (only same-method read + `SaveChanges` is listed, as uncertain).
|
|
9
|
+
2. **C# call graph gaps**: overload resolution by argument type, delegates / lambdas / extension methods, call chains longer than one level. Without Roslyn these stay approximate; edges carry `confirmed` / `probable` / `candidate`.
|
|
12
10
|
|
|
13
11
|
## Deferred (not worth doing yet)
|
|
14
12
|
|
|
15
13
|
| Item | Why deferred | Revisit when |
|
|
16
14
|
|---|---|---|
|
|
15
|
+
| Real embedding model as default | Measured 0.4.0 on a real 30k-chunk C# codebase with Chinese comments, 12 paraphrased queries (6 zh, 6 en): hashing MRR 0.48 / hit@5 6, `all-MiniLM-L6-v2` 0.29 / 6, `paraphrase-multilingual-MiniLM-L12-v2` (int8) 0.41 / 6; full index 128 s vs 35 / 43 min. Benchmark: MiniLM +0.08-0.13 MRR, multilingual -0.01..+0.03. Domain jargon ("split a lot into two") was missed by all three | A model trained on code + Chinese, or embedding becomes incremental enough that branch switches stay under a minute |
|
|
17
16
|
| ANN index | Exact search is not the bottleneck. Measured (Python stdlib copies, linear growth): 17k chunks cold CLI search 0.35 s / warm 0.13 s / index 10 s; 52k chunks 0.75 s / 0.36 s / 28 s; 104k chunks 1.09 s / 0.73 s / 54 s, 372 MB RSS, 729 MB database. At 104k chunks the cosine product plus top-k takes about 12 ms; loading the vectors out of SQLite (fetch 0.31 s, join 0.12 s) is about half of a warm query | Exact cosine itself becomes slow, which is far beyond 1M chunks |
|
|
18
17
|
| Vector matrix cache | Cheap and measurable: it removes about half of a warm query at 104k chunks (about 0.8 s -> 0.3 s). Not needed below ~50k chunks | A real project passes ~50k chunks. Keep the matrix in the MCP server process, and/or store it as a memory-mapped `.npy` next to the database so the CLI benefits too (invalidate on any index change) |
|
|
19
18
|
| `codebase_trace_execution` | Needs reliable call / data-flow relations; today's usage edges are name matches | Call graph gaps (item 3) are narrowed; today edges carry a confidence level |
|
|
@@ -23,6 +22,8 @@ Priorities come from the benchmark (`benchmarks/run_benchmark.py`, baseline in `
|
|
|
23
22
|
|
|
24
23
|
## Done
|
|
25
24
|
|
|
25
|
+
- Data access: `StringBuilder` SQL, EF LINQ on `DbSet<T>` members (resolved after indexing against declared DbSets), `programs` roll-up from accessors to controller actions; `relations Type.Method`; call graph resolves `Lazy<T>.Value` and `foreach (var x in ...)` element types; programs roll-up walks from an implementation to its interface method's callers and lists Razor pages (`Url.Action`, AngularJS `ControllerPath + 'Action'`); a typed identifier's scope includes files named after it and ranks names matching the rest of the query first ("WD0096 存檔" -> SetRecord); EF `SaveChanges` on tracked entities listed as possible writes; `.ai_context/extra_roots`; `.ai_context/ignore`, minified files skipped; C# namespace imports only link files whose types are used; piped CLI output is UTF-8
|
|
26
|
+
|
|
26
27
|
- SQL / data-access relations: `data_access` table and tool (`data-access` CLI, `codebase_get_data_access` MCP); T-SQL read / write / exec, C# SQL strings and Dapper / SqlCommand / EF sinks, EF table mappings, indirect writers through procedures, dynamic SQL reported as `uncertain`; search routes "who writes / reads / calls X" to it. Benchmark `sql_dataflow` Recall@5 0.72 -> 1.00, new `data_access` queries 0.67 -> 1.00
|
|
27
28
|
- C# symbol-level calls: using / field / constructor-injection / local type inference, interface dispatch, confidence levels. `relations Save` no longer shares one reference list across four unrelated methods
|
|
28
29
|
- Domain-term query expansion (`local_codebase_indexer/synonyms.py`, extendable with `.ai_context/synonyms.json`); `abbreviation` Recall@5 0.67 -> 1.00, `cross_lingual` 0.50 -> 1.00 (small, author-written sets: optimistic). ONNX pooling / query-prefix options
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Command line interface: ``
|
|
1
|
+
"""Command line interface: ``local-codebase-indexer <tool_name> --params`` (index.md section 4)."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -59,14 +59,14 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
def _fix_stdio(serving: bool) -> None:
|
|
62
|
-
"""MCP
|
|
63
|
-
keep its encoding but never crash on a character it cannot show."""
|
|
62
|
+
"""MCP and piped output (an AI assistant calling the CLI) are UTF-8; the Windows locale codec (cp950, cp1252)
|
|
63
|
+
would garble Chinese. On a console, keep its encoding but never crash on a character it cannot show."""
|
|
64
64
|
for name in ("stdin", "stdout", "stderr"):
|
|
65
65
|
stream = getattr(sys, name, None)
|
|
66
66
|
if stream is None or not hasattr(stream, "reconfigure"):
|
|
67
67
|
continue
|
|
68
68
|
try:
|
|
69
|
-
if serving:
|
|
69
|
+
if serving or (name != "stdin" and not stream.isatty()):
|
|
70
70
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
71
71
|
elif name != "stdin":
|
|
72
72
|
stream.reconfigure(errors="replace")
|
{local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/core.py
RENAMED
|
@@ -117,6 +117,7 @@ class Indexer:
|
|
|
117
117
|
dbmod.set_meta(conn, "embedder", self.embedder.name)
|
|
118
118
|
dbmod.set_meta(conn, "index_version", dbmod.INDEX_VERSION)
|
|
119
119
|
conn.commit()
|
|
120
|
+
conn.execute("VACUUM") # return the old index's pages to the file system
|
|
120
121
|
|
|
121
122
|
existing = {r["file_path"]: (r["last_modified"], r["md5_hash"]) for r in
|
|
122
123
|
conn.execute("SELECT file_path, last_modified, md5_hash FROM files")}
|
|
@@ -140,6 +141,7 @@ class Indexer:
|
|
|
140
141
|
stats.warnings.append(f"{sf.rel_path}: {exc}")
|
|
141
142
|
continue
|
|
142
143
|
if b"\x00" in data[:8192]:
|
|
144
|
+
seen.discard(sf.rel_path) # drop any earlier entry for it
|
|
143
145
|
stats.skipped += 1
|
|
144
146
|
continue
|
|
145
147
|
md5 = hashlib.md5(data).hexdigest()
|
|
@@ -173,6 +175,8 @@ class Indexer:
|
|
|
173
175
|
if changed or force:
|
|
174
176
|
resolve_dependencies(conn)
|
|
175
177
|
resolve_calls(conn)
|
|
178
|
+
resolve_page_actions(conn)
|
|
179
|
+
resolve_ef_access(conn)
|
|
176
180
|
self._record_run_meta()
|
|
177
181
|
conn.commit()
|
|
178
182
|
stats.chunks = conn.execute("SELECT COUNT(*) FROM chunk_metadata").fetchone()[0]
|
|
@@ -273,6 +277,82 @@ def _resolve_relative(source: str, spec: str, files: set[str]) -> str | None:
|
|
|
273
277
|
return None
|
|
274
278
|
|
|
275
279
|
|
|
280
|
+
def _view_route(page_path: str) -> tuple[str, str | None]:
|
|
281
|
+
"""(controller, area) a Razor page belongs to: ``Areas/<area>/Views/<controller>/x.cshtml`` or
|
|
282
|
+
``Views/<controller>/x.cshtml``; area None outside Areas."""
|
|
283
|
+
parts = page_path.split("/")
|
|
284
|
+
ctrl = parts[parts.index("Views") + 1] if "Views" in parts[:-2] else ""
|
|
285
|
+
area = parts[parts.index("Areas") + 1] if "Areas" in parts[:-1] else None
|
|
286
|
+
return ctrl, area
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def resolve_page_actions(conn: sqlite3.Connection) -> None:
|
|
290
|
+
"""Add ``calls`` edges from Razor pages to the controller actions their ``Url.Action(..)`` targets."""
|
|
291
|
+
by_action: dict[str, list[sqlite3.Row]] = defaultdict(list)
|
|
292
|
+
for r in conn.execute("SELECT id, symbol_name, qualified_name, file_path FROM symbols WHERE symbol_type='method' "
|
|
293
|
+
"AND qualified_name LIKE '%Controller.%'"):
|
|
294
|
+
by_action[r["symbol_name"]].append(r)
|
|
295
|
+
rows = []
|
|
296
|
+
for r in conn.execute("SELECT r.file_path, r.name, r.line, r.owner FROM raw_refs r WHERE r.kind='action'"):
|
|
297
|
+
ctrl, action = r["name"].split("|", 1)
|
|
298
|
+
own_ctrl, area = _view_route(r["file_path"])
|
|
299
|
+
ctrl = ctrl or own_ctrl
|
|
300
|
+
if not ctrl:
|
|
301
|
+
continue
|
|
302
|
+
cands = [m for m in by_action.get(action, ()) if m["qualified_name"].split(".")[-2] == ctrl + "Controller"]
|
|
303
|
+
if area is not None:
|
|
304
|
+
same_area = [m for m in cands if f"/Areas/{area}/" in "/" + m["file_path"]]
|
|
305
|
+
else:
|
|
306
|
+
same_area = [m for m in cands if "/Areas/" not in "/" + m["file_path"]]
|
|
307
|
+
cands = same_area or cands
|
|
308
|
+
for m in cands:
|
|
309
|
+
rows.append((r["owner"], m["id"], r["file_path"], r["line"], "confirmed" if len(cands) == 1 else "probable",
|
|
310
|
+
"Url.Action in page"))
|
|
311
|
+
conn.executemany("INSERT INTO calls(caller_id, callee_id, file_path, line, confidence, reason) VALUES(?,?,?,?,?,?)",
|
|
312
|
+
rows)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
_DBSET = re.compile(r"\bI?DbSet<\s*([\w.]+)\s*>")
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def resolve_ef_access(conn: sqlite3.Connection) -> None:
|
|
319
|
+
"""Turn ``ef_raw`` candidates whose member is a declared ``DbSet<T>`` into ``ef_linq`` table accesses.
|
|
320
|
+
Table = the EF mapping of T (``ToTable`` / ``[Table]``) when there is one, else the DbSet property name.
|
|
321
|
+
A method that reads a DbSet and calls ``SaveChanges`` without an explicit Add / Remove on it may update tracked
|
|
322
|
+
entities: that becomes an ``uncertain`` write (listed as possible, never as a confirmed writer)."""
|
|
323
|
+
conn.execute("DELETE FROM data_access WHERE via='ef_linq'")
|
|
324
|
+
dbsets: dict[str, str] = {}
|
|
325
|
+
for r in conn.execute("SELECT symbol_name, signature FROM symbols WHERE symbol_type='property' AND signature LIKE '%DbSet<%'"):
|
|
326
|
+
m = _DBSET.search(r[1] or "")
|
|
327
|
+
if m:
|
|
328
|
+
dbsets[r[0].lower()] = m.group(1).split(".")[-1]
|
|
329
|
+
if not dbsets:
|
|
330
|
+
return
|
|
331
|
+
mapped = {r[0].split(".")[-1]: r[1] for r in conn.execute(
|
|
332
|
+
"SELECT detail, target_name FROM data_access WHERE via='ef_mapping' AND detail != ''")}
|
|
333
|
+
rows = []
|
|
334
|
+
for r in conn.execute("SELECT * FROM data_access WHERE via='ef_raw'").fetchall():
|
|
335
|
+
entity = dbsets.get(r["target_key"][3:])
|
|
336
|
+
if entity is None:
|
|
337
|
+
continue
|
|
338
|
+
table = mapped.get(entity, r["target_name"])
|
|
339
|
+
rows.append((r["file_path"], r["source_id"], r["line"], table.lower(), table, "", "table", r["access"],
|
|
340
|
+
r["op"], "ef_linq", "probable", r["evidence"], entity))
|
|
341
|
+
savers = {r[0]: (r[1], r[2]) for r in conn.execute(
|
|
342
|
+
"SELECT source_id, line, evidence FROM data_access WHERE via='ef_raw' AND target_key='ef:#savechanges' "
|
|
343
|
+
"AND source_id IS NOT NULL")}
|
|
344
|
+
written = {(r[1], r[3]) for r in rows if r[7] == "write"}
|
|
345
|
+
for r in list(rows):
|
|
346
|
+
if r[7] == "read" and r[1] in savers and (r[1], r[3]) not in written:
|
|
347
|
+
written.add((r[1], r[3]))
|
|
348
|
+
line, evidence = savers[r[1]]
|
|
349
|
+
rows.append((r[0], r[1], line, r[3], r[4], "", "table", "write", "update", "ef_linq", "uncertain",
|
|
350
|
+
f"tracked {r[4]} entities + {evidence}", r[12]))
|
|
351
|
+
conn.executemany(
|
|
352
|
+
"INSERT INTO data_access(file_path, source_id, line, target_key, target_name, qualifier, target_kind, access,"
|
|
353
|
+
" op, via, confidence, evidence, detail) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", rows)
|
|
354
|
+
|
|
355
|
+
|
|
276
356
|
def resolve_dependencies(conn: sqlite3.Connection) -> None:
|
|
277
357
|
"""Rebuild the file-level dependency graph from raw_refs and symbols."""
|
|
278
358
|
conn.execute("DELETE FROM dependencies")
|
|
@@ -289,6 +369,15 @@ def resolve_dependencies(conn: sqlite3.Connection) -> None:
|
|
|
289
369
|
for r in conn.execute("SELECT symbol_name, file_path FROM symbols WHERE symbol_type='namespace'"):
|
|
290
370
|
ns_files[r[0]].add(r[1])
|
|
291
371
|
|
|
372
|
+
# a namespace import only links to the files in it whose types the importing file names
|
|
373
|
+
file_types: dict[str, set[str]] = defaultdict(set)
|
|
374
|
+
for name, paths in type_files.items():
|
|
375
|
+
for p in paths:
|
|
376
|
+
file_types[p].add(name)
|
|
377
|
+
used: dict[str, set[str]] = defaultdict(set)
|
|
378
|
+
for r in conn.execute("SELECT file_path, name FROM raw_refs WHERE kind IN ('ident', 'base')"):
|
|
379
|
+
used[r[0]].add(r[1])
|
|
380
|
+
|
|
292
381
|
deps: set[tuple[str, str, str]] = set()
|
|
293
382
|
|
|
294
383
|
for r in conn.execute("SELECT file_path, name FROM raw_refs WHERE kind='import'"):
|
|
@@ -299,7 +388,7 @@ def resolve_dependencies(conn: sqlite3.Connection) -> None:
|
|
|
299
388
|
if t:
|
|
300
389
|
targets.add(t)
|
|
301
390
|
elif name in ns_files:
|
|
302
|
-
targets |= ns_files[name]
|
|
391
|
+
targets |= {t for t in ns_files[name] if file_types[t] & used[src]}
|
|
303
392
|
else:
|
|
304
393
|
slug = name.replace(".", "/")
|
|
305
394
|
for cand in (slug + ".py", slug + "/__init__.py"):
|
{local_codebase_indexer-0.3.2 → local_codebase_indexer-0.4.0}/local_codebase_indexer/cscalls.py
RENAMED
|
@@ -14,8 +14,14 @@ _SCOPES = {
|
|
|
14
14
|
"method_declaration", "constructor_declaration", "destructor_declaration", "operator_declaration",
|
|
15
15
|
"indexer_declaration", "property_declaration",
|
|
16
16
|
}
|
|
17
|
-
_TASK = re.compile(r"^(?:[\w.]+\.)?(?:Task|ValueTask)\s*<(.+)>$")
|
|
17
|
+
_TASK = re.compile(r"^(?:[\w.]+\.)?(?:Task|ValueTask|Lazy)\s*<(.+)>$")
|
|
18
18
|
_OUTER = re.compile(r"^([\w.]+)\s*(?:<.*>)?$", re.S)
|
|
19
|
+
_ELEM = re.compile(r"^(?:[\w.]+\.)?(?:IEnumerable|IList|List|ICollection|IReadOnlyList|IReadOnlyCollection|HashSet|ISet"
|
|
20
|
+
r"|IQueryable)\s*<(.+)>$", re.S)
|
|
21
|
+
# LINQ calls whose result has the same element type as their source
|
|
22
|
+
_KEEP_ELEM = {"Where", "OrderBy", "OrderByDescending", "ThenBy", "ThenByDescending", "ToList", "ToArray", "AsEnumerable",
|
|
23
|
+
"Skip", "Take", "Distinct", "Reverse"}
|
|
24
|
+
_CLASSES = {"class_declaration", "struct_declaration", "record_declaration", "interface_declaration"}
|
|
19
25
|
|
|
20
26
|
|
|
21
27
|
@dataclass
|
|
@@ -33,7 +39,7 @@ class CallSite:
|
|
|
33
39
|
|
|
34
40
|
def norm_type(text: str) -> str:
|
|
35
41
|
"""Outer type name without namespace, generic arguments, nullability or array brackets.
|
|
36
|
-
``Task<T>`` / ``ValueTask<T>`` become ``T`` (the type a caller works with after ``await``)."""
|
|
42
|
+
``Task<T>`` / ``ValueTask<T>`` / ``Lazy<T>`` become ``T`` (the type a caller works with after ``await`` / ``.Value``)."""
|
|
37
43
|
t = (text or "").strip()
|
|
38
44
|
t = re.sub(r"^(?:ref|out|in|readonly|const|params)\s+", "", t)
|
|
39
45
|
t = t.replace("global::", "")
|
|
@@ -49,6 +55,15 @@ def norm_type(text: str) -> str:
|
|
|
49
55
|
return m.group(1).split(".")[-1] if m else ""
|
|
50
56
|
|
|
51
57
|
|
|
58
|
+
def elem_type(text: str) -> str:
|
|
59
|
+
"""Element type of a collection type (``IEnumerable<T>``, ``List<T>``, ``T[]`` ...), else ""."""
|
|
60
|
+
t = re.sub(r"\?$", "", (text or "").strip())
|
|
61
|
+
if t.endswith("[]"):
|
|
62
|
+
return norm_type(t[:-2])
|
|
63
|
+
m = _ELEM.match(t)
|
|
64
|
+
return norm_type(m.group(1)) if m else ""
|
|
65
|
+
|
|
66
|
+
|
|
52
67
|
def _text(n, src: bytes) -> str:
|
|
53
68
|
return src[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
|
54
69
|
|
|
@@ -94,10 +109,59 @@ class _Calls:
|
|
|
94
109
|
spans.sort(key=lambda t: (t[0], t[1].start))
|
|
95
110
|
self.spans = [s for _, s in spans]
|
|
96
111
|
self.out: list[CallSite] = []
|
|
112
|
+
self._field_types: dict[int, dict[str, str]] = {}
|
|
97
113
|
|
|
98
114
|
def text(self, n) -> str:
|
|
99
115
|
return _text(n, self.src)
|
|
100
116
|
|
|
117
|
+
def _class_member_types(self, scope) -> dict[str, str]:
|
|
118
|
+
"""Declared type text of the fields and properties of the class enclosing ``scope``."""
|
|
119
|
+
cls = scope.parent
|
|
120
|
+
while cls is not None and cls.type not in _CLASSES:
|
|
121
|
+
cls = cls.parent
|
|
122
|
+
if cls is None:
|
|
123
|
+
return {}
|
|
124
|
+
if cls.id not in self._field_types:
|
|
125
|
+
types: dict[str, str] = {}
|
|
126
|
+
body = cls.child_by_field_name("body")
|
|
127
|
+
for m in (body.named_children if body is not None else ()):
|
|
128
|
+
if m.type == "property_declaration":
|
|
129
|
+
t, nm = m.child_by_field_name("type"), m.child_by_field_name("name")
|
|
130
|
+
if t is not None and nm is not None:
|
|
131
|
+
types[self.text(nm)] = self.text(t)
|
|
132
|
+
elif m.type == "field_declaration":
|
|
133
|
+
for vd in m.named_children:
|
|
134
|
+
if vd.type != "variable_declaration":
|
|
135
|
+
continue
|
|
136
|
+
t = vd.child_by_field_name("type")
|
|
137
|
+
for d in vd.named_children:
|
|
138
|
+
if d.type != "variable_declarator":
|
|
139
|
+
continue
|
|
140
|
+
ident = next((c for c in d.named_children if c.type == "identifier"), None)
|
|
141
|
+
if t is not None and ident is not None:
|
|
142
|
+
types[self.text(ident)] = self.text(t)
|
|
143
|
+
self._field_types[cls.id] = types
|
|
144
|
+
return self._field_types[cls.id]
|
|
145
|
+
|
|
146
|
+
def _foreach_elem(self, expr, raw: dict, scope) -> str:
|
|
147
|
+
"""Element type of ``foreach (var x in <expr>)`` when <expr> is a collection variable, field or property,
|
|
148
|
+
optionally filtered by LINQ calls that keep the element type."""
|
|
149
|
+
n = self._unwrap(expr)
|
|
150
|
+
while n is not None and n.type == "invocation_expression":
|
|
151
|
+
func = n.child_by_field_name("function")
|
|
152
|
+
if func is None or func.type != "member_access_expression" or self._method_name(func, self.text) not in _KEEP_ELEM:
|
|
153
|
+
return ""
|
|
154
|
+
n = self._unwrap(func.child_by_field_name("expression"))
|
|
155
|
+
if n is not None and n.type == "member_access_expression":
|
|
156
|
+
inner = n.child_by_field_name("expression")
|
|
157
|
+
if inner is None or inner.type != "this_expression":
|
|
158
|
+
return ""
|
|
159
|
+
n = n.child_by_field_name("name")
|
|
160
|
+
if n is None or n.type != "identifier":
|
|
161
|
+
return ""
|
|
162
|
+
name = self.text(n)
|
|
163
|
+
return elem_type(raw.get(name) or self._class_member_types(scope).get(name, ""))
|
|
164
|
+
|
|
101
165
|
def locate(self, line: int):
|
|
102
166
|
for s in self.spans:
|
|
103
167
|
if s.start <= line <= s.end:
|
|
@@ -140,6 +204,7 @@ class _Calls:
|
|
|
140
204
|
|
|
141
205
|
def collect_locals(self, scope) -> dict:
|
|
142
206
|
env: dict[str, str] = {}
|
|
207
|
+
raw: dict[str, str] = {} # declared type text, for foreach element types
|
|
143
208
|
plist = scope.child_by_field_name("parameters")
|
|
144
209
|
if plist is not None:
|
|
145
210
|
for p in plist.named_children:
|
|
@@ -147,6 +212,7 @@ class _Calls:
|
|
|
147
212
|
t, nm = p.child_by_field_name("type"), p.child_by_field_name("name")
|
|
148
213
|
if t is not None and nm is not None:
|
|
149
214
|
self._set(env, self.text(nm), norm_type(self.text(t)))
|
|
215
|
+
raw[self.text(nm)] = self.text(t)
|
|
150
216
|
stack = list(scope.children)
|
|
151
217
|
while stack:
|
|
152
218
|
n = stack.pop()
|
|
@@ -167,10 +233,16 @@ class _Calls:
|
|
|
167
233
|
init = c
|
|
168
234
|
typ = declared if not implicit else (self._infer_initializer(init, env) if init is not None else "")
|
|
169
235
|
self._set(env, self.text(ident), typ)
|
|
236
|
+
if not implicit:
|
|
237
|
+
raw[self.text(ident)] = self.text(tnode)
|
|
170
238
|
elif ty == "foreach_statement":
|
|
171
239
|
tnode, left = n.child_by_field_name("type"), n.child_by_field_name("left")
|
|
172
240
|
if left is not None and left.type == "identifier":
|
|
173
|
-
|
|
241
|
+
if tnode is None or tnode.type == "implicit_type":
|
|
242
|
+
right = n.child_by_field_name("right")
|
|
243
|
+
typ = self._foreach_elem(right, raw, scope) if right is not None else ""
|
|
244
|
+
else:
|
|
245
|
+
typ = norm_type(self.text(tnode))
|
|
174
246
|
self._set(env, self.text(left), typ)
|
|
175
247
|
elif ty == "catch_declaration":
|
|
176
248
|
kids = list(n.named_children)
|
|
@@ -220,6 +292,10 @@ class _Calls:
|
|
|
220
292
|
nm = n.child_by_field_name("name")
|
|
221
293
|
if inner is not None and inner.type == "this_expression" and nm is not None:
|
|
222
294
|
return "member", self.text(nm), "", ""
|
|
295
|
+
if inner is not None and inner.type == "identifier" and nm is not None and self.text(nm) == "Value":
|
|
296
|
+
# ponytail: x.Value is read as x (Lazy<T> / Nullable<T> already normalised to T); any other type's
|
|
297
|
+
# .Value resolves to that type, usually outside the index, so the call gets no edge
|
|
298
|
+
return self._classify_receiver(inner, env, depth)
|
|
223
299
|
return "expr", "", "", ""
|
|
224
300
|
if t == "object_creation_expression":
|
|
225
301
|
tn = n.child_by_field_name("type")
|