java-codebase-rag 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ast_java.py +28 -57
- build_ast_graph.py +486 -262
- graph_enrich.py +13 -35
- java_codebase_rag/cli.py +10 -10
- java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +3 -3
- java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +5 -5
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/METADATA +10 -10
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/RECORD +18 -18
- java_index_flow_lancedb.py +95 -21
- ladybug_queries.py +45 -51
- mcp_v2.py +9 -17
- path_filtering.py +24 -19
- pr_analysis.py +3 -4
- search_lancedb.py +2 -2
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/WHEEL +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/entry_points.txt +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/licenses/LICENSE +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/top_level.txt +0 -0
graph_enrich.py
CHANGED
|
@@ -23,7 +23,7 @@ import sys
|
|
|
23
23
|
from dataclasses import dataclass, field, replace
|
|
24
24
|
from functools import lru_cache
|
|
25
25
|
from pathlib import Path
|
|
26
|
-
from typing import Any
|
|
26
|
+
from typing import Any, TypeVar
|
|
27
27
|
from ast_java import (
|
|
28
28
|
AnnotationRef,
|
|
29
29
|
JavaFileAst,
|
|
@@ -820,7 +820,15 @@ def _route_path_atom(raw_value: str, value_kind: str | None) -> tuple[str, str,
|
|
|
820
820
|
return "", "constant_ref", 0.7, False
|
|
821
821
|
|
|
822
822
|
|
|
823
|
-
|
|
823
|
+
_HINT = TypeVar("_HINT")
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def _hint_lookup(ann: AnnotationRef, hints: dict[str, _HINT]) -> _HINT | None:
|
|
827
|
+
"""Resolve a brownfield hint by qualified name, then simple name, then suffix.
|
|
828
|
+
|
|
829
|
+
Shared by route / http-client / async-producer hint resolution; the three
|
|
830
|
+
former copies differed only in the hint value type.
|
|
831
|
+
"""
|
|
824
832
|
q = ann.qualified.strip()
|
|
825
833
|
if q in hints:
|
|
826
834
|
return hints[q]
|
|
@@ -1118,7 +1126,7 @@ def resolve_routes_for_method(
|
|
|
1118
1126
|
|
|
1119
1127
|
# ----- Step 2: Layer B — annotation route hints -----
|
|
1120
1128
|
for _is_m, ann in combined_anns:
|
|
1121
|
-
hint =
|
|
1129
|
+
hint = _hint_lookup(ann, overrides.annotation_to_route_hint)
|
|
1122
1130
|
if hint is None:
|
|
1123
1131
|
continue
|
|
1124
1132
|
working.append(
|
|
@@ -1172,36 +1180,6 @@ def resolve_routes_for_method(
|
|
|
1172
1180
|
return working
|
|
1173
1181
|
|
|
1174
1182
|
|
|
1175
|
-
def _client_hint_lookup(
|
|
1176
|
-
ann: AnnotationRef,
|
|
1177
|
-
hints: dict[str, HttpClientHint],
|
|
1178
|
-
) -> HttpClientHint | None:
|
|
1179
|
-
q = ann.qualified.strip()
|
|
1180
|
-
if q in hints:
|
|
1181
|
-
return hints[q]
|
|
1182
|
-
if ann.name in hints:
|
|
1183
|
-
return hints[ann.name]
|
|
1184
|
-
for k, h in sorted(hints.items(), key=lambda kv: kv[0]):
|
|
1185
|
-
if k.endswith("." + ann.name):
|
|
1186
|
-
return h
|
|
1187
|
-
return None
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
def _async_hint_lookup(
|
|
1191
|
-
ann: AnnotationRef,
|
|
1192
|
-
hints: dict[str, AsyncProducerHint],
|
|
1193
|
-
) -> AsyncProducerHint | None:
|
|
1194
|
-
q = ann.qualified.strip()
|
|
1195
|
-
if q in hints:
|
|
1196
|
-
return hints[q]
|
|
1197
|
-
if ann.name in hints:
|
|
1198
|
-
return hints[ann.name]
|
|
1199
|
-
for k, h in sorted(hints.items(), key=lambda kv: kv[0]):
|
|
1200
|
-
if k.endswith("." + ann.name):
|
|
1201
|
-
return h
|
|
1202
|
-
return None
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
1183
|
def _call_from_http_hint(
|
|
1206
1184
|
*,
|
|
1207
1185
|
hint: HttpClientHint,
|
|
@@ -1296,7 +1274,7 @@ def resolve_http_client_for_method(
|
|
|
1296
1274
|
anchor = builtin_http[0] if builtin_http else (layer_c_src[0] if layer_c_src else None)
|
|
1297
1275
|
|
|
1298
1276
|
for _is_m, ann in combined_anns:
|
|
1299
|
-
hint =
|
|
1277
|
+
hint = _hint_lookup(ann, overrides.annotation_to_http_client_hint)
|
|
1300
1278
|
if hint is None:
|
|
1301
1279
|
continue
|
|
1302
1280
|
brownfield_calls.append(
|
|
@@ -1388,7 +1366,7 @@ def resolve_async_producer_for_method(
|
|
|
1388
1366
|
anchor = builtin_async[0] if builtin_async else (layer_c_src[0] if layer_c_src else None)
|
|
1389
1367
|
|
|
1390
1368
|
for _is_m, ann in combined_anns:
|
|
1391
|
-
hint =
|
|
1369
|
+
hint = _hint_lookup(ann, overrides.annotation_to_async_producer_hint)
|
|
1392
1370
|
if hint is None:
|
|
1393
1371
|
continue
|
|
1394
1372
|
brownfield_calls.append(
|
java_codebase_rag/cli.py
CHANGED
|
@@ -251,7 +251,7 @@ def _startup_hints(cfg: ResolvedOperatorConfig) -> None:
|
|
|
251
251
|
|
|
252
252
|
def _add_index_embedding_flags(p: argparse.ArgumentParser) -> None:
|
|
253
253
|
p.add_argument("--source-root", type=str, default=None, help="Java repository root (default: cwd)")
|
|
254
|
-
p.add_argument("--index-dir", type=str, default=None, help="Index directory (Lance +
|
|
254
|
+
p.add_argument("--index-dir", type=str, default=None, help="Index directory (Lance + LadybugDB + cocoindex state)")
|
|
255
255
|
p.add_argument("--embedding-model", type=str, default=None, help="Override SBERT_MODEL / YAML embedding.model")
|
|
256
256
|
p.add_argument("--embedding-device", type=str, default=None, help="Override SBERT_DEVICE / YAML embedding.device")
|
|
257
257
|
|
|
@@ -536,7 +536,7 @@ def _cmd_reprocess(args: argparse.Namespace) -> int:
|
|
|
536
536
|
_emit_reprocess_outcome(payload, selective_tty_mode="graph" if ok else None)
|
|
537
537
|
return _reprocess_exit_code(payload)
|
|
538
538
|
|
|
539
|
-
import server # lazy: pulls sentence_transformers/torch/lancedb/
|
|
539
|
+
import server # lazy: pulls sentence_transformers/torch/lancedb/ladybug
|
|
540
540
|
|
|
541
541
|
result = asyncio.run(
|
|
542
542
|
server.run_refresh_pipeline(
|
|
@@ -715,7 +715,7 @@ def _cmd_unresolved_calls_list(args: argparse.Namespace) -> int:
|
|
|
715
715
|
from ladybug_queries import LadybugGraph # lazy
|
|
716
716
|
|
|
717
717
|
if not LadybugGraph.exists():
|
|
718
|
-
_emit({"success": False, "message": "
|
|
718
|
+
_emit({"success": False, "message": "LadybugDB graph not found"})
|
|
719
719
|
return 1
|
|
720
720
|
graph = LadybugGraph.get()
|
|
721
721
|
rows = graph.list_unresolved_call_sites(
|
|
@@ -736,7 +736,7 @@ def _cmd_unresolved_calls_stats(args: argparse.Namespace) -> int:
|
|
|
736
736
|
from ladybug_queries import LadybugGraph # lazy
|
|
737
737
|
|
|
738
738
|
if not LadybugGraph.exists():
|
|
739
|
-
_emit({"success": False, "message": "
|
|
739
|
+
_emit({"success": False, "message": "LadybugDB graph not found"})
|
|
740
740
|
return 1
|
|
741
741
|
graph = LadybugGraph.get()
|
|
742
742
|
buckets = graph.stats_unresolved_call_sites(by=args.by)
|
|
@@ -761,7 +761,7 @@ def _cmd_analyze_pr(args: argparse.Namespace) -> int:
|
|
|
761
761
|
from ladybug_queries import LadybugGraph # lazy
|
|
762
762
|
|
|
763
763
|
if not LadybugGraph.exists():
|
|
764
|
-
_emit({"success": False, "message": "
|
|
764
|
+
_emit({"success": False, "message": "LadybugDB graph not found"})
|
|
765
765
|
return 1
|
|
766
766
|
graph = LadybugGraph.get()
|
|
767
767
|
report = pr_analysis.analyze_pr_pipeline(graph, diff_text)
|
|
@@ -801,7 +801,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
801
801
|
help="Create a fresh index from a Java repository.",
|
|
802
802
|
description=(
|
|
803
803
|
"First-time index creation. Refuses if the resolved index directory "
|
|
804
|
-
"already contains a
|
|
804
|
+
"already contains a LadybugDB graph or Lance tables. Exit 2 on refusal."
|
|
805
805
|
),
|
|
806
806
|
)
|
|
807
807
|
_add_index_embedding_flags(init)
|
|
@@ -870,7 +870,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
870
870
|
increment = subparsers.add_parser(
|
|
871
871
|
"increment",
|
|
872
872
|
help="Pick up changes since the last index update.",
|
|
873
|
-
description="Runs cocoindex catch-up and incremental
|
|
873
|
+
description="Runs cocoindex catch-up and incremental LadybugDB graph update. Use --vectors-only to skip graph update.",
|
|
874
874
|
)
|
|
875
875
|
_add_index_embedding_flags(increment)
|
|
876
876
|
_add_verbosity_flags(increment)
|
|
@@ -883,9 +883,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
883
883
|
|
|
884
884
|
reprocess = subparsers.add_parser(
|
|
885
885
|
"reprocess",
|
|
886
|
-
help="Rebuild vectors and/or
|
|
886
|
+
help="Rebuild vectors and/or LadybugDB (default: both full phases).",
|
|
887
887
|
description=(
|
|
888
|
-
"Default: full Lance reprocess (cocoindex --full-reprocess) then full
|
|
888
|
+
"Default: full Lance reprocess (cocoindex --full-reprocess) then full LadybugDB graph rebuild. "
|
|
889
889
|
"Use --vectors-only or --graph-only to run a single phase (mutually exclusive)."
|
|
890
890
|
),
|
|
891
891
|
)
|
|
@@ -907,7 +907,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
907
907
|
erase = subparsers.add_parser(
|
|
908
908
|
"erase",
|
|
909
909
|
help="Delete the index from disk.",
|
|
910
|
-
description="Runs cocoindex drop, removes
|
|
910
|
+
description="Runs cocoindex drop, removes LadybugDB, and drops Lance tables. Requires --yes or TTY confirmation.",
|
|
911
911
|
)
|
|
912
912
|
_add_index_embedding_flags(erase)
|
|
913
913
|
erase.add_argument("--yes", action="store_true", help="Confirm destructive deletion (required in CI)")
|
|
@@ -151,15 +151,15 @@ Simple types in parentheses; generics erased. No spaces after commas. No-arg: `(
|
|
|
151
151
|
|
|
152
152
|
### Shared NodeFilter
|
|
153
153
|
|
|
154
|
-
For `find`, `filter` is required — `{}` means no predicates. **Strict frame:** unknown keys or inapplicable populated fields → `success=false
|
|
154
|
+
For `find`, `filter` is required — `{}` means no predicates. **Strict frame:** unknown keys or inapplicable populated fields → `success=false`; invalid enum values (e.g. wrong case) are rejected earlier at the schema layer with the valid set listed.
|
|
155
155
|
|
|
156
156
|
| Keys | Applies to |
|
|
157
157
|
| ---- | ---------- |
|
|
158
158
|
| `microservice`, `module` | All kinds |
|
|
159
159
|
| `role`, `exclude_roles`, `annotation`, `capability`, `fqn_prefix`, `symbol_kind`, `symbol_kinds` | **symbol** |
|
|
160
160
|
| `http_method`, `path_prefix`, `framework` | **route** |
|
|
161
|
-
| `client_kind`, `target_service`, `target_path_prefix`, `http_method` | **client** |
|
|
162
|
-
| `producer_kind`, `topic_prefix` | **producer** |
|
|
161
|
+
| `source_layer`, `client_kind`, `target_service`, `target_path_prefix`, `http_method` | **client** |
|
|
162
|
+
| `source_layer`, `producer_kind`, `topic_prefix` | **producer** |
|
|
163
163
|
|
|
164
164
|
No wildcards in prefix fields — use `search(query=…)` for fuzzy text.
|
|
165
165
|
|
|
@@ -125,15 +125,15 @@ Use these strings **verbatim** in `neighbors(..., edge_types=[...])`.
|
|
|
125
125
|
|
|
126
126
|
### NodeFilter (`find`, `search.filter`, `neighbors.filter`)
|
|
127
127
|
|
|
128
|
-
For `find`, `filter` is required — `{}` means no predicates. **Strict frame:** unknown keys or inapplicable populated fields → `success=false
|
|
128
|
+
For `find`, `filter` is required — `{}` means no predicates. **Strict frame:** unknown keys or inapplicable populated fields → `success=false`; invalid enum values (e.g. wrong case) are rejected earlier at the schema layer with the valid set listed.
|
|
129
129
|
|
|
130
130
|
| Applicable to | Keys |
|
|
131
131
|
| ------------- | ---- |
|
|
132
132
|
| All kinds | `microservice`, `module` |
|
|
133
133
|
| **symbol** only | `role`, `exclude_roles`, `annotation`, `capability`, `fqn_prefix`, `symbol_kind`, `symbol_kinds` |
|
|
134
134
|
| **route** only | `http_method`, `path_prefix`, `framework` |
|
|
135
|
-
| **client** only | `client_kind`, `target_service`, `target_path_prefix`, `http_method` |
|
|
136
|
-
| **producer** only | `producer_kind`, `topic_prefix` |
|
|
135
|
+
| **client** only | `source_layer`, `client_kind`, `target_service`, `target_path_prefix`, `http_method` |
|
|
136
|
+
| **producer** only | `source_layer`, `producer_kind`, `topic_prefix` |
|
|
137
137
|
|
|
138
138
|
No wildcards in prefix fields — use `search(query=…)` for ranked text.
|
|
139
139
|
|
|
@@ -166,8 +166,8 @@ Exclude `DTO`, `OTHER`, `MAPPER` with `exclude_roles` when tracing business logi
|
|
|
166
166
|
|
|
167
167
|
**Symbol kinds:** `class`, `interface`, `enum`, `record`, `annotation`, `method`, `constructor`.
|
|
168
168
|
|
|
169
|
-
**Route frameworks:** `spring_mvc`, `webflux
|
|
170
|
-
**Client kinds:** `feign_method`, `rest_template`, `web_client`. **Producer kinds:** `kafka_send`, `stream_bridge_send`.
|
|
169
|
+
**Route frameworks:** `spring_mvc`, `webflux`. (Route *kinds* are `http_endpoint`, `http_consumer`, `kafka_topic`, `rabbit_queue`, `jms_destination`, `stream_binding`.)
|
|
170
|
+
**Client kinds:** `feign_method`, `rest_template`, `web_client`. **Producer kinds:** `kafka_send`, `stream_bridge_send`. **Source layers (client/producer):** `builtin`, `layer_a_meta`, `layer_b_ann`, `layer_b_fqn`, `layer_c_source`.
|
|
171
171
|
**Match types:** `cross_service`, `intra_service`, `ambiguous`, `phantom`, `unresolved`.
|
|
172
172
|
|
|
173
173
|
---
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: java-codebase-rag
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.5
|
|
4
4
|
Summary: MCP server for semantic + structural search over Java codebases
|
|
5
5
|
Author: HumanBean17
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,7 +44,7 @@ Dynamic: license-file
|
|
|
44
44
|
|
|
45
45
|
A graph-native code intelligence layer for Java microservice estates, exposed to LLM agents via the **Model Context Protocol (MCP)**.
|
|
46
46
|
|
|
47
|
-
The system extracts a deterministic property graph from Java source (tree-sitter), stores it in **
|
|
47
|
+
The system extracts a deterministic property graph from Java source (tree-sitter), stores it in **LadybugDB** (graph) alongside a **LanceDB** vector index (chunks), and exposes a deliberately small MCP surface — **five tools**: `search`, `find`, `describe`, `neighbors`, `resolve` — that collapse onto three primitive agent operations: **locate**, **inspect**, **walk**.
|
|
48
48
|
|
|
49
49
|
> **What this MCP is:** a **GPS for code navigation**, not a reasoning engine.
|
|
50
50
|
> Agents use a simple loop:
|
|
@@ -63,9 +63,9 @@ For the design rationale, the GPS metaphor, and the full ontology, see [`docs/pa
|
|
|
63
63
|
|
|
64
64
|
Generic code-search tools (grep, ctags, vector-only RAG) hit a ceiling on real Java microservice estates: they find files but lose the structure that makes a Spring/JAX-RS system navigable. This project is built around five choices that target that gap.
|
|
65
65
|
|
|
66
|
-
- **Hybrid RAG + GraphRAG, not either-or.** Semantic recall (LanceDB chunk vectors) and structural navigation (
|
|
66
|
+
- **Hybrid RAG + GraphRAG, not either-or.** Semantic recall (LanceDB chunk vectors) and structural navigation (LadybugDB property graph) are composed in one surface. `search` finds candidate nodes by meaning; `neighbors` walks the exact edge you care about (`CALLS`, `IMPLEMENTS`, `INJECTS`, `EXPOSES`, …). The agent picks the right primitive per step instead of being forced into pure-vector or pure-symbol search.
|
|
67
67
|
|
|
68
|
-
- **A Java-tuned role model.** Symbols are labelled with stereotypes inferred from Spring and JAX-RS conventions — `CONTROLLER`, `SERVICE`, `REPOSITORY`, `
|
|
68
|
+
- **A Java-tuned role model.** Symbols are labelled with stereotypes inferred from Spring and JAX-RS conventions — `CONTROLLER`, `SERVICE`, `REPOSITORY`, `COMPONENT`, `CONFIG`, `ENTITY`, `CLIENT`, `MAPPER`, `DTO`. Agents can ask "list controllers" or "who injects this repository" directly, instead of grep-ing for `@RestController` and hoping for the best. Roles drive both filtering (`find` with a `NodeFilter`) and ranking.
|
|
69
69
|
|
|
70
70
|
- **Ranking specialized for Java codebases.** The composite ranker is aware of role, microservice, and FQN structure — not a generic BM25. A search for `"chat ingress"` surfaces controllers before utility classes; a search scoped to one microservice doesn't drown in matches from the other 19. Defaults are tuned on the bank-chat fixture and exposed in `docs/CONFIGURATION.md` for per-repo overrides.
|
|
71
71
|
|
|
@@ -113,7 +113,7 @@ All indexing lifecycle commands (`init`, `increment`, `reprocess`, `install`, `u
|
|
|
113
113
|
|
|
114
114
|
If you prefer manual configuration, see [`docs/JAVA-CODEBASE-RAG-CLI.md`](./docs/JAVA-CODEBASE-RAG-CLI.md) for the full CLI reference.
|
|
115
115
|
|
|
116
|
-
> **Stability disclaimer.** This package does **not** promise backward compatibility. MCP tool contracts, env vars, Lance/
|
|
116
|
+
> **Stability disclaimer.** This package does **not** promise backward compatibility. MCP tool contracts, env vars, Lance/LadybugDB schemas, config files, and Python APIs may change without a deprecation period. Track `main` and rebuild indexes when ontology or embedding settings change.
|
|
117
117
|
|
|
118
118
|
---
|
|
119
119
|
|
|
@@ -126,7 +126,7 @@ This repo ships a small multi-module Spring fixture under [`tests/bank-chat-syst
|
|
|
126
126
|
git clone https://github.com/HumanBean17/java-codebase-rag
|
|
127
127
|
cd java-codebase-rag
|
|
128
128
|
|
|
129
|
-
# 2. Build the index (Lance vectors +
|
|
129
|
+
# 2. Build the index (Lance vectors + LadybugDB graph). First run downloads the
|
|
130
130
|
# embedding model (~90 MB) and takes ~30-60s on the fixture.
|
|
131
131
|
java-codebase-rag init --source-root tests/bank-chat-system --index-dir /tmp/bank-chat-index
|
|
132
132
|
|
|
@@ -141,7 +141,7 @@ Smoke-test the index with two checks (`search_lancedb` ships with the package):
|
|
|
141
141
|
JAVA_CODEBASE_RAG_INDEX_DIR=/tmp/bank-chat-index \
|
|
142
142
|
python -m search_lancedb "chat ingress controller" --table java --limit 3
|
|
143
143
|
|
|
144
|
-
# Vector + graph expansion — proves
|
|
144
|
+
# Vector + graph expansion — proves LadybugDB is wired in
|
|
145
145
|
JAVA_CODEBASE_RAG_INDEX_DIR=/tmp/bank-chat-index \
|
|
146
146
|
python -m search_lancedb "chat ingress controller" --table java --limit 3 \
|
|
147
147
|
--graph-expand --expand-depth 2
|
|
@@ -241,8 +241,8 @@ Run `java-codebase-rag --help` to list grouped subcommands. Operator playbook wi
|
|
|
241
241
|
| Setup | `install` | Interactive setup wizard: config, MCP registration, skill/agent deployment, indexing. |
|
|
242
242
|
| Setup | `update` | Refresh shipped artifacts (skill, agent, MCP entry) + incremental Lance/graph catch-up after pip upgrade. |
|
|
243
243
|
| Lifecycle | `init` | First-time index. Refuses if artifacts already exist. |
|
|
244
|
-
| Lifecycle | `increment` | CocoIndex catch-up + incremental
|
|
245
|
-
| Lifecycle | `reprocess` | Full Lance +
|
|
244
|
+
| Lifecycle | `increment` | CocoIndex catch-up + incremental LadybugDB update. `--vectors-only` for Lance only. |
|
|
245
|
+
| Lifecycle | `reprocess` | Full Lance + LadybugDB rebuild. `--vectors-only` / `--graph-only` for a single phase. |
|
|
246
246
|
| Lifecycle | `erase` | Delete index artifacts. Requires `--yes` or TTY confirm. |
|
|
247
247
|
| Introspection | `meta`, `tables`, `diagnose-ignore`, `unresolved-calls` | Health, table listing, ignore-layer diagnostics, receiver-failure call sites. |
|
|
248
248
|
| Analysis | `analyze-pr` | Blast-radius / risk from a unified diff. |
|
|
@@ -277,7 +277,7 @@ python3 -m venv .venv
|
|
|
277
277
|
|
|
278
278
|
The `cocoindex` package powers lifecycle commands that run the indexer (`init`, `increment`, `reprocess`, `erase`). Search and MCP navigation do not invoke it directly.
|
|
279
279
|
|
|
280
|
-
The default embedding model is `sentence-transformers/all-MiniLM-L6-v2` (downloaded on first `init`). Override via the `
|
|
280
|
+
The default embedding model is `sentence-transformers/all-MiniLM-L6-v2` (downloaded on first `init`). Override via the `SBERT_MODEL` env var — see [`docs/CONFIGURATION.md` §1](./docs/CONFIGURATION.md#1-environment-variables).
|
|
281
281
|
|
|
282
282
|
---
|
|
283
283
|
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
ast_java.py,sha256=
|
|
1
|
+
ast_java.py,sha256=Paee3ZV9G5iy8LqfkVq0Ah_1fV0k632oS5JPkiWzwB8,99206
|
|
2
2
|
brownfield_events.py,sha256=yxXkKDgMb3VPtaiakGzncHM_EGnda8xIue6w90yYp8s,2055
|
|
3
|
-
build_ast_graph.py,sha256=
|
|
3
|
+
build_ast_graph.py,sha256=jubb9Ex_6B3ktsInqNkk-EksBtQjofoFhE92lmHr308,169659
|
|
4
4
|
chunk_heuristics.py,sha256=aQk2NOKxzUdqoUAJUO3G3LE0MN_bYZWNLQ0tkmj5uts,1813
|
|
5
|
-
graph_enrich.py,sha256=
|
|
5
|
+
graph_enrich.py,sha256=W_OQK7YRU1K8wi-vfrxZWUd-EErTqWx840UBrhVpNsM,62788
|
|
6
6
|
index_common.py,sha256=HT6FKHFJ084eFvd3fR1j8z8gf4eWoPHVW8GXLpw464I,285
|
|
7
|
-
java_index_flow_lancedb.py,sha256=
|
|
7
|
+
java_index_flow_lancedb.py,sha256=JHdRWDZoZ3wnxi8NNG2ck8zEmSALHB2UVB1N8TSvlmg,24109
|
|
8
8
|
java_index_v1_common.py,sha256=nF1KrSqboF_RRvWerG9knRRFmWwsrG_CvhgnsoZ8KqA,1154
|
|
9
9
|
java_ontology.py,sha256=71bCLDNvMy0SpZPzSR5apJ0qJXNd6y5ggkLdBEw_PFo,16682
|
|
10
|
-
ladybug_queries.py,sha256=
|
|
10
|
+
ladybug_queries.py,sha256=7vSP7WsUKQYXHFenBMlCdpUFed39h3oul-WRkf55YVc,90330
|
|
11
11
|
mcp_hints.py,sha256=3swh05LSiWur3tm3-yssndBsLxIxFhy501kBtJI8jJ0,42509
|
|
12
|
-
mcp_v2.py,sha256=
|
|
13
|
-
path_filtering.py,sha256
|
|
14
|
-
pr_analysis.py,sha256=
|
|
15
|
-
search_lancedb.py,sha256=
|
|
12
|
+
mcp_v2.py,sha256=S8PiVGDlyI7cvTmeMdQobhKUxWqKoY2YxXXr4pu9lQI,80348
|
|
13
|
+
path_filtering.py,sha256=R--XzI51LXBu5IBKMCnJWbkNr6I5d-SDmltyQQnWco0,17674
|
|
14
|
+
pr_analysis.py,sha256=zrmZZD5yotJtM02Kif6_jgI_oeformOao793akp0N6Y,18394
|
|
15
|
+
search_lancedb.py,sha256=ga_qySeCzA6hCibxUPXpCbTxW9mCjTdwirMhMhfNCz4,36801
|
|
16
16
|
server.py,sha256=DcJwocSknBy0vwsUvYiC5hr-hrD_rRT-u9_DmuRTC9k,35035
|
|
17
17
|
java_codebase_rag/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
18
18
|
java_codebase_rag/_fdlimit.py,sha256=WroFdfSNbcriKok6q8znTf74dqlznxea_1Fd5bHl_3o,1930
|
|
19
|
-
java_codebase_rag/cli.py,sha256=
|
|
19
|
+
java_codebase_rag/cli.py,sha256=TIrKNQMxu3_dSF2LXKsRKWY5RqRnSLQMrbKAoN4uQBQ,38698
|
|
20
20
|
java_codebase_rag/cli_format.py,sha256=CT7-xdwZ0bMCdP68_UOwkvm-mnLluU3LutlM-mDNk60,1839
|
|
21
21
|
java_codebase_rag/cli_progress.py,sha256=q6Wh97yzLGs1B8UFk_WAKivfQu7Y5RnUUE-T2YHWkIs,3237
|
|
22
22
|
java_codebase_rag/config.py,sha256=bfwYI4R8PU9YV_M4r8-03iaUZ_0TW-qN_NuhIsDXy2M,18769
|
|
@@ -24,11 +24,11 @@ java_codebase_rag/installer.py,sha256=sXsHPo24aoDFoTr0D_vYLg0MFdGAV2wdL05FqRaul6
|
|
|
24
24
|
java_codebase_rag/lance_optimize.py,sha256=25Rwj7HNO8F-35MxhFK6naqgbjd3H-T0zKb3pXB4H0s,9268
|
|
25
25
|
java_codebase_rag/pipeline.py,sha256=ydNktEGL1YniAjJsr37yKBo_bGV4cN_LTGVTmmrsrZw,14688
|
|
26
26
|
java_codebase_rag/progress.py,sha256=2IxdMALDM0wAQCyJrrfZ975zM_85C-4BfHxf4AtYifE,23212
|
|
27
|
-
java_codebase_rag/install_data/agents/explorer-rag-enhanced.md,sha256=
|
|
28
|
-
java_codebase_rag/install_data/skills/explore-codebase/SKILL.md,sha256=
|
|
29
|
-
java_codebase_rag-0.6.
|
|
30
|
-
java_codebase_rag-0.6.
|
|
31
|
-
java_codebase_rag-0.6.
|
|
32
|
-
java_codebase_rag-0.6.
|
|
33
|
-
java_codebase_rag-0.6.
|
|
34
|
-
java_codebase_rag-0.6.
|
|
27
|
+
java_codebase_rag/install_data/agents/explorer-rag-enhanced.md,sha256=BkdQpBEWqSdvGHgbqMdRb5CWfEiFRJK4Dgqbyal3l6s,14551
|
|
28
|
+
java_codebase_rag/install_data/skills/explore-codebase/SKILL.md,sha256=YkRnrM7Wh5E8raFjAW3RrN2V9-ov8upaGC3UdpSx6U8,12346
|
|
29
|
+
java_codebase_rag-0.6.5.dist-info/licenses/LICENSE,sha256=gxvtiHtuviR_q8ZAjWw-QTcF3DyPzg6ZY-lQrr8OPpw,1068
|
|
30
|
+
java_codebase_rag-0.6.5.dist-info/METADATA,sha256=jbKFqnBab9XoMAtIhfX-A9n0ztxhOEPrNcuH5G9qrEM,17242
|
|
31
|
+
java_codebase_rag-0.6.5.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
32
|
+
java_codebase_rag-0.6.5.dist-info/entry_points.txt,sha256=wsPZwot0Ui4JI3TIgW8LcbN8bNtKFbwQAlHAAJXfYgQ,117
|
|
33
|
+
java_codebase_rag-0.6.5.dist-info/top_level.txt,sha256=syQgi8XPBwY2ws_NZ1uRCxTf_s41NpshwEHNdcdnk3A,245
|
|
34
|
+
java_codebase_rag-0.6.5.dist-info/RECORD,,
|
java_index_flow_lancedb.py
CHANGED
|
@@ -16,6 +16,7 @@ Usage:
|
|
|
16
16
|
"""
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import asyncio
|
|
19
20
|
import inspect
|
|
20
21
|
import os
|
|
21
22
|
import sys
|
|
@@ -50,7 +51,7 @@ from java_index_v1_common import (
|
|
|
50
51
|
)
|
|
51
52
|
from path_filtering import LayeredIgnore
|
|
52
53
|
from ast_java import ONTOLOGY_VERSION, parse_java
|
|
53
|
-
from graph_enrich import enrich_chunk
|
|
54
|
+
from graph_enrich import collect_annotation_meta_chain, enrich_chunk, load_brownfield_overrides
|
|
54
55
|
|
|
55
56
|
# Older cocoindex (e.g. 1.0.0a43) uses ``tracked=False``; newer releases renamed
|
|
56
57
|
# the flag to ``detect_change`` (default False) and reject ``tracked``.
|
|
@@ -59,16 +60,21 @@ if "detect_change" in _ck_params:
|
|
|
59
60
|
PROJECT_ROOT = coco.ContextKey[Path]("java_lance_project_root")
|
|
60
61
|
LANCE_DB = coco.ContextKey("java_lance_async_conn")
|
|
61
62
|
EMBEDDER = coco.ContextKey[SentenceTransformerEmbedder]("java_lance_embedder")
|
|
63
|
+
IGNORE = coco.ContextKey[LayeredIgnore]("java_lance_layered_ignore")
|
|
62
64
|
elif "tracked" in _ck_params:
|
|
63
65
|
PROJECT_ROOT = coco.ContextKey[Path]("java_lance_project_root", tracked=False)
|
|
64
66
|
LANCE_DB = coco.ContextKey("java_lance_async_conn", tracked=False)
|
|
65
67
|
EMBEDDER = coco.ContextKey[SentenceTransformerEmbedder](
|
|
66
68
|
"java_lance_embedder", tracked=False
|
|
67
69
|
)
|
|
70
|
+
IGNORE = coco.ContextKey[LayeredIgnore](
|
|
71
|
+
"java_lance_layered_ignore", tracked=False
|
|
72
|
+
)
|
|
68
73
|
else:
|
|
69
74
|
PROJECT_ROOT = coco.ContextKey[Path]("java_lance_project_root")
|
|
70
75
|
LANCE_DB = coco.ContextKey("java_lance_async_conn")
|
|
71
76
|
EMBEDDER = coco.ContextKey[SentenceTransformerEmbedder]("java_lance_embedder")
|
|
77
|
+
IGNORE = coco.ContextKey[LayeredIgnore]("java_lance_layered_ignore")
|
|
72
78
|
|
|
73
79
|
splitter = RecursiveSplitter()
|
|
74
80
|
|
|
@@ -198,12 +204,12 @@ def _approximate_vectors_total(project_root: Path) -> int:
|
|
|
198
204
|
continue
|
|
199
205
|
# Java: **/*.java
|
|
200
206
|
if fn.endswith(".java"):
|
|
201
|
-
if not ignore.is_ignored(full)
|
|
207
|
+
if not ignore.is_ignored(full):
|
|
202
208
|
total += 1
|
|
203
209
|
continue
|
|
204
210
|
# SQL: **/src/main/resources/db/migration/*.sql
|
|
205
211
|
if fn.endswith(".sql") and "/db/migration/" in rel:
|
|
206
|
-
if not ignore.is_ignored(full)
|
|
212
|
+
if not ignore.is_ignored(full):
|
|
207
213
|
total += 1
|
|
208
214
|
continue
|
|
209
215
|
# YAML: **/src/main/resources/application*.yml / .yaml
|
|
@@ -214,7 +220,7 @@ def _approximate_vectors_total(project_root: Path) -> int:
|
|
|
214
220
|
# total below the actual done count. The ``rel``-based
|
|
215
221
|
# ``"/src/main/resources/"`` gate stays (full path component).
|
|
216
222
|
if fn.endswith((".yml", ".yaml")) and fn.startswith("application") and "/src/main/resources/" in rel:
|
|
217
|
-
if not ignore.is_ignored(full)
|
|
223
|
+
if not ignore.is_ignored(full):
|
|
218
224
|
total += 1
|
|
219
225
|
return total
|
|
220
226
|
|
|
@@ -291,6 +297,7 @@ async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]
|
|
|
291
297
|
trust_remote_code=True,
|
|
292
298
|
)
|
|
293
299
|
builder.provide(EMBEDDER, embedder)
|
|
300
|
+
builder.provide(IGNORE, LayeredIgnore(root))
|
|
294
301
|
|
|
295
302
|
uri = str(index_dir)
|
|
296
303
|
|
|
@@ -306,6 +313,40 @@ async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]
|
|
|
306
313
|
yield
|
|
307
314
|
|
|
308
315
|
|
|
316
|
+
def _parse_and_enrich_java(
|
|
317
|
+
content_bytes: bytes,
|
|
318
|
+
chunks: list[Any],
|
|
319
|
+
rel: str,
|
|
320
|
+
project_root: Path,
|
|
321
|
+
) -> list[Any]:
|
|
322
|
+
"""Parse one Java file and enrich every chunk, off the event loop.
|
|
323
|
+
|
|
324
|
+
Returns a list of :class:`graph_enrich.ChunkEnrichment` aligned 1:1 with
|
|
325
|
+
``chunks``. Intended to run via ``asyncio.to_thread`` from
|
|
326
|
+
``process_java_file`` (vectors perf lever #2): while the worker thread
|
|
327
|
+
parses + enriches, the event loop is free to drive other files and keep the
|
|
328
|
+
embedder's batching queue fed.
|
|
329
|
+
|
|
330
|
+
Thread-safety: ``parse_java`` uses a per-thread tree-sitter ``Parser``
|
|
331
|
+
(see ``ast_java._parser``), so it is safe to call concurrently from these
|
|
332
|
+
worker threads — including the transitive ``parse_java`` that ``enrich_chunk``
|
|
333
|
+
triggers via ``collect_annotation_meta_chain`` → ``_collect_annotation_decl_index``.
|
|
334
|
+
``enrich_chunk`` is otherwise pure-Python over the now-immutable AST; its
|
|
335
|
+
``lru_cache`` reads are thread-safe under the GIL.
|
|
336
|
+
"""
|
|
337
|
+
ast = parse_java(content_bytes)
|
|
338
|
+
return [
|
|
339
|
+
enrich_chunk(
|
|
340
|
+
ast,
|
|
341
|
+
chunk_start_byte=ch.start.byte_offset,
|
|
342
|
+
chunk_end_byte=ch.end.byte_offset,
|
|
343
|
+
file_path=rel,
|
|
344
|
+
project_root=project_root,
|
|
345
|
+
)
|
|
346
|
+
for ch in chunks
|
|
347
|
+
]
|
|
348
|
+
|
|
349
|
+
|
|
309
350
|
@coco.fn(memo=True)
|
|
310
351
|
async def process_java_file(
|
|
311
352
|
file: localfs.File,
|
|
@@ -313,7 +354,8 @@ async def process_java_file(
|
|
|
313
354
|
) -> None:
|
|
314
355
|
embedder = coco.use_context(EMBEDDER)
|
|
315
356
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
316
|
-
|
|
357
|
+
ignore = coco.use_context(IGNORE)
|
|
358
|
+
if ignore.is_ignored((project_root / file.file_path.path).resolve()):
|
|
317
359
|
return
|
|
318
360
|
try:
|
|
319
361
|
content = await file.read_text()
|
|
@@ -326,6 +368,9 @@ async def process_java_file(
|
|
|
326
368
|
|
|
327
369
|
language = detect_code_language(filename=file.file_path.path.name) or "text"
|
|
328
370
|
cs, mn, ov = JAVA_CHUNK
|
|
371
|
+
# ``splitter.split`` stays inline: the module-level ``RecursiveSplitter``
|
|
372
|
+
# shares one Rust object, so keeping split on the event loop preserves its
|
|
373
|
+
# existing single-threaded access (no new cross-file concurrency hazard).
|
|
329
374
|
chunks = splitter.split(
|
|
330
375
|
content,
|
|
331
376
|
cs,
|
|
@@ -335,18 +380,21 @@ async def process_java_file(
|
|
|
335
380
|
)
|
|
336
381
|
rel = file.file_path.path.as_posix()
|
|
337
382
|
content_bytes = content.encode("utf-8", errors="replace")
|
|
338
|
-
ast = parse_java(content_bytes)
|
|
339
383
|
|
|
340
|
-
|
|
384
|
+
# (vectors perf lever #2) parse + enrich off the event loop so the loop can
|
|
385
|
+
# keep the embedder's batching queue fed while this file is being parsed.
|
|
386
|
+
# parse_java is thread-safe (per-thread tree-sitter Parser in ast_java).
|
|
387
|
+
enrichments = await asyncio.to_thread(
|
|
388
|
+
_parse_and_enrich_java, content_bytes, chunks, rel, project_root
|
|
389
|
+
)
|
|
390
|
+
# (vectors perf lever #1) embed all chunks concurrently so the batched
|
|
391
|
+
# embedder groups them into one ``model.encode(...)`` (max_batch_size=64)
|
|
392
|
+
# instead of N serial batch-of-1 calls. Dominant win for ``increment``
|
|
393
|
+
# (few changed files → little cross-file concurrency → otherwise no batching).
|
|
394
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
395
|
+
|
|
396
|
+
for ch, enrich, emb in zip(chunks, enrichments, embeddings):
|
|
341
397
|
rs, re = chunk_key_range(ch)
|
|
342
|
-
enrich = enrich_chunk(
|
|
343
|
-
ast,
|
|
344
|
-
chunk_start_byte=ch.start.byte_offset,
|
|
345
|
-
chunk_end_byte=ch.end.byte_offset,
|
|
346
|
-
file_path=rel,
|
|
347
|
-
project_root=project_root,
|
|
348
|
-
)
|
|
349
|
-
emb = await embedder.embed(ch.text)
|
|
350
398
|
table.declare_row(
|
|
351
399
|
row=JavaLanceChunk(
|
|
352
400
|
id=str(uuid.uuid4()),
|
|
@@ -379,7 +427,8 @@ async def process_sql_file(
|
|
|
379
427
|
) -> None:
|
|
380
428
|
embedder = coco.use_context(EMBEDDER)
|
|
381
429
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
382
|
-
|
|
430
|
+
ignore = coco.use_context(IGNORE)
|
|
431
|
+
if ignore.is_ignored((project_root / file.file_path.path).resolve()):
|
|
383
432
|
return
|
|
384
433
|
try:
|
|
385
434
|
content = await file.read_text()
|
|
@@ -401,9 +450,11 @@ async def process_sql_file(
|
|
|
401
450
|
)
|
|
402
451
|
rel = file.file_path.path.as_posix()
|
|
403
452
|
|
|
404
|
-
|
|
453
|
+
# (vectors perf lever #1) embed chunks concurrently → batched encode.
|
|
454
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
455
|
+
|
|
456
|
+
for ch, emb in zip(chunks, embeddings):
|
|
405
457
|
rs, re = chunk_key_range(ch)
|
|
406
|
-
emb = await embedder.embed(ch.text)
|
|
407
458
|
table.declare_row(
|
|
408
459
|
row=SqlLanceChunk(
|
|
409
460
|
id=str(uuid.uuid4()),
|
|
@@ -425,7 +476,8 @@ async def process_yaml_file(
|
|
|
425
476
|
) -> None:
|
|
426
477
|
embedder = coco.use_context(EMBEDDER)
|
|
427
478
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
428
|
-
|
|
479
|
+
ignore = coco.use_context(IGNORE)
|
|
480
|
+
if ignore.is_ignored((project_root / file.file_path.path).resolve()):
|
|
429
481
|
return
|
|
430
482
|
try:
|
|
431
483
|
content = await file.read_text()
|
|
@@ -448,9 +500,11 @@ async def process_yaml_file(
|
|
|
448
500
|
)
|
|
449
501
|
rel = file.file_path.path.as_posix()
|
|
450
502
|
|
|
451
|
-
|
|
503
|
+
# (vectors perf lever #1) embed chunks concurrently → batched encode.
|
|
504
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
505
|
+
|
|
506
|
+
for ch, emb in zip(chunks, embeddings):
|
|
452
507
|
rs, re = chunk_key_range(ch)
|
|
453
|
-
emb = await embedder.embed(ch.text)
|
|
454
508
|
table.declare_row(
|
|
455
509
|
row=YamlLanceChunk(
|
|
456
510
|
id=str(uuid.uuid4()),
|
|
@@ -501,6 +555,26 @@ async def app_main() -> None:
|
|
|
501
555
|
)
|
|
502
556
|
|
|
503
557
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
558
|
+
# Warm per-project enrichment caches ONCE on the event-loop thread, BEFORE
|
|
559
|
+
# coco.mount_each fans files into worker threads. collect_annotation_meta_chain
|
|
560
|
+
# and load_brownfield_overrides are lru_cached per (resolved) project root;
|
|
561
|
+
# without warming, the first wave of concurrent process_java_file worker
|
|
562
|
+
# threads each cold-miss and redundantly walk+parse the ENTIRE project (a
|
|
563
|
+
# thundering herd that would offset the embedding-batching win on large
|
|
564
|
+
# repos — perf lever #2 made enrich concurrent). With warming, every worker
|
|
565
|
+
# hits a populated cache (lru_cache reads are thread-safe). Key derivation
|
|
566
|
+
# mirrors enrich_chunk exactly so the warmed entries are the ones workers hit.
|
|
567
|
+
try:
|
|
568
|
+
load_brownfield_overrides(project_root)
|
|
569
|
+
try:
|
|
570
|
+
prs = str(Path(project_root).resolve())
|
|
571
|
+
except OSError:
|
|
572
|
+
prs = str(project_root)
|
|
573
|
+
collect_annotation_meta_chain(prs)
|
|
574
|
+
except Exception:
|
|
575
|
+
# Warm-up must never break indexing — a failure just means workers
|
|
576
|
+
# cold-miss lazily (the pre-warming behavior). Swallow and continue.
|
|
577
|
+
pass
|
|
504
578
|
_ignore = LayeredIgnore(project_root)
|
|
505
579
|
_walk_excludes = _ignore.cocoindex_excluded_patterns()
|
|
506
580
|
# Emit ONE approximate total so the parent's renderer can show a determinate
|