java-codebase-rag 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ast_java.py +28 -57
- build_ast_graph.py +118 -26
- graph_enrich.py +13 -35
- java_codebase_rag/cli.py +97 -24
- java_codebase_rag/cli_format.py +8 -35
- java_codebase_rag/cli_progress.py +17 -21
- java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +3 -3
- java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +5 -5
- java_codebase_rag/installer.py +180 -43
- java_codebase_rag/lance_optimize.py +125 -51
- java_codebase_rag/pipeline.py +71 -23
- java_codebase_rag/progress.py +570 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.4.dist-info}/METADATA +13 -10
- java_codebase_rag-0.6.4.dist-info/RECORD +34 -0
- java_index_flow_lancedb.py +238 -18
- ladybug_queries.py +45 -51
- mcp_v2.py +9 -17
- path_filtering.py +14 -17
- pr_analysis.py +3 -4
- search_lancedb.py +2 -2
- server.py +75 -9
- java_codebase_rag-0.6.2.dist-info/RECORD +0 -33
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.4.dist-info}/WHEEL +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.4.dist-info}/entry_points.txt +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.4.dist-info}/licenses/LICENSE +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.4.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
ast_java.py,sha256=Paee3ZV9G5iy8LqfkVq0Ah_1fV0k632oS5JPkiWzwB8,99206
|
|
2
|
+
brownfield_events.py,sha256=yxXkKDgMb3VPtaiakGzncHM_EGnda8xIue6w90yYp8s,2055
|
|
3
|
+
build_ast_graph.py,sha256=vXWUuNC_k6-yKv6XawlHCXiC_dTNs1RoSKdZTXYBxgA,157374
|
|
4
|
+
chunk_heuristics.py,sha256=aQk2NOKxzUdqoUAJUO3G3LE0MN_bYZWNLQ0tkmj5uts,1813
|
|
5
|
+
graph_enrich.py,sha256=W_OQK7YRU1K8wi-vfrxZWUd-EErTqWx840UBrhVpNsM,62788
|
|
6
|
+
index_common.py,sha256=HT6FKHFJ084eFvd3fR1j8z8gf4eWoPHVW8GXLpw464I,285
|
|
7
|
+
java_index_flow_lancedb.py,sha256=HClxupO2uHpGw9GNVIDs24bjoCQzg3VLbOZoA0hE_Gc,23761
|
|
8
|
+
java_index_v1_common.py,sha256=nF1KrSqboF_RRvWerG9knRRFmWwsrG_CvhgnsoZ8KqA,1154
|
|
9
|
+
java_ontology.py,sha256=71bCLDNvMy0SpZPzSR5apJ0qJXNd6y5ggkLdBEw_PFo,16682
|
|
10
|
+
ladybug_queries.py,sha256=7vSP7WsUKQYXHFenBMlCdpUFed39h3oul-WRkf55YVc,90330
|
|
11
|
+
mcp_hints.py,sha256=3swh05LSiWur3tm3-yssndBsLxIxFhy501kBtJI8jJ0,42509
|
|
12
|
+
mcp_v2.py,sha256=S8PiVGDlyI7cvTmeMdQobhKUxWqKoY2YxXXr4pu9lQI,80348
|
|
13
|
+
path_filtering.py,sha256=cN_znZRXIY1CfTIWlfiOrnNqafb2BZs00oDaqjTxmFE,17169
|
|
14
|
+
pr_analysis.py,sha256=zrmZZD5yotJtM02Kif6_jgI_oeformOao793akp0N6Y,18394
|
|
15
|
+
search_lancedb.py,sha256=ga_qySeCzA6hCibxUPXpCbTxW9mCjTdwirMhMhfNCz4,36801
|
|
16
|
+
server.py,sha256=DcJwocSknBy0vwsUvYiC5hr-hrD_rRT-u9_DmuRTC9k,35035
|
|
17
|
+
java_codebase_rag/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
18
|
+
java_codebase_rag/_fdlimit.py,sha256=WroFdfSNbcriKok6q8znTf74dqlznxea_1Fd5bHl_3o,1930
|
|
19
|
+
java_codebase_rag/cli.py,sha256=TIrKNQMxu3_dSF2LXKsRKWY5RqRnSLQMrbKAoN4uQBQ,38698
|
|
20
|
+
java_codebase_rag/cli_format.py,sha256=CT7-xdwZ0bMCdP68_UOwkvm-mnLluU3LutlM-mDNk60,1839
|
|
21
|
+
java_codebase_rag/cli_progress.py,sha256=q6Wh97yzLGs1B8UFk_WAKivfQu7Y5RnUUE-T2YHWkIs,3237
|
|
22
|
+
java_codebase_rag/config.py,sha256=bfwYI4R8PU9YV_M4r8-03iaUZ_0TW-qN_NuhIsDXy2M,18769
|
|
23
|
+
java_codebase_rag/installer.py,sha256=sXsHPo24aoDFoTr0D_vYLg0MFdGAV2wdL05FqRaul6E,52861
|
|
24
|
+
java_codebase_rag/lance_optimize.py,sha256=25Rwj7HNO8F-35MxhFK6naqgbjd3H-T0zKb3pXB4H0s,9268
|
|
25
|
+
java_codebase_rag/pipeline.py,sha256=ydNktEGL1YniAjJsr37yKBo_bGV4cN_LTGVTmmrsrZw,14688
|
|
26
|
+
java_codebase_rag/progress.py,sha256=2IxdMALDM0wAQCyJrrfZ975zM_85C-4BfHxf4AtYifE,23212
|
|
27
|
+
java_codebase_rag/install_data/agents/explorer-rag-enhanced.md,sha256=BkdQpBEWqSdvGHgbqMdRb5CWfEiFRJK4Dgqbyal3l6s,14551
|
|
28
|
+
java_codebase_rag/install_data/skills/explore-codebase/SKILL.md,sha256=YkRnrM7Wh5E8raFjAW3RrN2V9-ov8upaGC3UdpSx6U8,12346
|
|
29
|
+
java_codebase_rag-0.6.4.dist-info/licenses/LICENSE,sha256=gxvtiHtuviR_q8ZAjWw-QTcF3DyPzg6ZY-lQrr8OPpw,1068
|
|
30
|
+
java_codebase_rag-0.6.4.dist-info/METADATA,sha256=FR0VQ3dS_OS2v-n_zxOqcE0pZsHRvhDAiOGLse0pAoI,17242
|
|
31
|
+
java_codebase_rag-0.6.4.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
32
|
+
java_codebase_rag-0.6.4.dist-info/entry_points.txt,sha256=wsPZwot0Ui4JI3TIgW8LcbN8bNtKFbwQAlHAAJXfYgQ,117
|
|
33
|
+
java_codebase_rag-0.6.4.dist-info/top_level.txt,sha256=syQgi8XPBwY2ws_NZ1uRCxTf_s41NpshwEHNdcdnk3A,245
|
|
34
|
+
java_codebase_rag-0.6.4.dist-info/RECORD,,
|
java_index_flow_lancedb.py
CHANGED
|
@@ -16,12 +16,16 @@ Usage:
|
|
|
16
16
|
"""
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import asyncio
|
|
19
20
|
import inspect
|
|
20
21
|
import os
|
|
22
|
+
import sys
|
|
23
|
+
import threading
|
|
21
24
|
import uuid
|
|
22
25
|
from collections.abc import AsyncIterator
|
|
23
26
|
from contextlib import asynccontextmanager
|
|
24
27
|
from dataclasses import dataclass
|
|
28
|
+
from fnmatch import fnmatch
|
|
25
29
|
from pathlib import Path
|
|
26
30
|
from typing import Annotated, Any
|
|
27
31
|
|
|
@@ -47,7 +51,7 @@ from java_index_v1_common import (
|
|
|
47
51
|
)
|
|
48
52
|
from path_filtering import LayeredIgnore
|
|
49
53
|
from ast_java import ONTOLOGY_VERSION, parse_java
|
|
50
|
-
from graph_enrich import enrich_chunk
|
|
54
|
+
from graph_enrich import collect_annotation_meta_chain, enrich_chunk, load_brownfield_overrides
|
|
51
55
|
|
|
52
56
|
# Older cocoindex (e.g. 1.0.0a43) uses ``tracked=False``; newer releases renamed
|
|
53
57
|
# the flag to ``detect_change`` (default False) and reject ``tracked``.
|
|
@@ -84,6 +88,138 @@ splitter = RecursiveSplitter()
|
|
|
84
88
|
_NUM_TXN_BEFORE_OPTIMIZE = 10**12
|
|
85
89
|
|
|
86
90
|
|
|
91
|
+
# --- Vectors-phase progress emission (JCIRAG_PROGRESS kind=vectors) -----------
|
|
92
|
+
#
|
|
93
|
+
# The flow runs in a CHILD cocoindex process; it prints structured progress to
|
|
94
|
+
# its stderr and the parent (pipeline._popen_capturing_stderr /
|
|
95
|
+
# cli_progress.accumulate_and_relay_subprocess_streams) parses it via
|
|
96
|
+
# ProgressRelay and feeds the renderer. The flow CANNOT know when all files are
|
|
97
|
+
# done (cocoindex offers no "all files done" hook in the flow), so it emits:
|
|
98
|
+
# - ONE ``total=N status=running`` line from ``app_main`` (approximate
|
|
99
|
+
# pre-walk: matcher includes + LayeredIgnore), and
|
|
100
|
+
# - per-file ``done=k status=running`` ticks (throttled every ~25 files) from
|
|
101
|
+
# ``process_*_file`` (shared atomic counter).
|
|
102
|
+
# The PARENT emits the terminal ``status=done``/``failed`` vectors event on
|
|
103
|
+
# cocoindex exit (drives clamp-on-completion + phase transition to Optimize).
|
|
104
|
+
|
|
105
|
+
# Per-file tick cadence: bound stderr volume on huge trees without making the
|
|
106
|
+
# bar feel stale. Every 25th file (and the modulo boundary is enough — the
|
|
107
|
+
# parent clamps to total on the terminal event anyway).
|
|
108
|
+
_VECTORS_TICK_EVERY = 25
|
|
109
|
+
|
|
110
|
+
# Thread-safe counter: cocoindex may call process_*_file concurrently
|
|
111
|
+
# (mount_each parallelism is implementation-defined). A module-level lock guards
|
|
112
|
+
# both the counter and the emission so two threads never interleave a tick.
|
|
113
|
+
_vectors_done_lock = threading.Lock()
|
|
114
|
+
_vectors_done_count = 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _emit_vectors_progress(
|
|
118
|
+
*,
|
|
119
|
+
done: int | None = None,
|
|
120
|
+
total: int | None = None,
|
|
121
|
+
status: str = "running",
|
|
122
|
+
elapsed_s: float | None = None,
|
|
123
|
+
) -> None:
|
|
124
|
+
"""Emit one ``JCIRAG_PROGRESS kind=vectors …`` line to stderr (flushed).
|
|
125
|
+
|
|
126
|
+
Field order is fixed (kind, done, total, status, elapsed_s) so the parser
|
|
127
|
+
and tests can pin substrings. Omitted fields are simply absent.
|
|
128
|
+
"""
|
|
129
|
+
fields = ["kind=vectors"]
|
|
130
|
+
if done is not None:
|
|
131
|
+
fields.append(f"done={done}")
|
|
132
|
+
if total is not None:
|
|
133
|
+
fields.append(f"total={total}")
|
|
134
|
+
fields.append(f"status={status}")
|
|
135
|
+
if elapsed_s is not None:
|
|
136
|
+
fields.append(f"elapsed_s={elapsed_s:.2f}")
|
|
137
|
+
print("JCIRAG_PROGRESS " + " ".join(fields), file=sys.stderr, flush=True)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _tick_vectors_done() -> None:
|
|
141
|
+
"""Increment the shared per-file counter and emit a throttled ``done=k`` tick.
|
|
142
|
+
|
|
143
|
+
Called once per successfully-processed file (after the ignore / empty
|
|
144
|
+
early-returns). The tick is emitted every ``_VECTORS_TICK_EVERY`` files so
|
|
145
|
+
stderr volume stays bounded on huge trees; the parent clamps to total on
|
|
146
|
+
the terminal event, so the exact tick cadence is not load-bearing.
|
|
147
|
+
"""
|
|
148
|
+
global _vectors_done_count
|
|
149
|
+
with _vectors_done_lock:
|
|
150
|
+
_vectors_done_count += 1
|
|
151
|
+
n = _vectors_done_count
|
|
152
|
+
if n % _VECTORS_TICK_EVERY != 0:
|
|
153
|
+
return
|
|
154
|
+
# Emit under the lock: the docstring above promises the lock guards both
|
|
155
|
+
# the counter AND the emission, so two concurrent ticks can't emit their
|
|
156
|
+
# ``done=N`` lines out of order. Contention is negligible (fires every
|
|
157
|
+
# ~25 files).
|
|
158
|
+
_emit_vectors_progress(done=n, status="running")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _approximate_vectors_total(project_root: Path) -> int:
|
|
162
|
+
"""Reproduce the matchers' include globs + LayeredIgnore for an approximate total.
|
|
163
|
+
|
|
164
|
+
The flow applies two filtering layers: (1) ``PatternFilePathMatcher``
|
|
165
|
+
excludes at walk time via ``LayeredIgnore.cocoindex_excluded_patterns()``,
|
|
166
|
+
then (2) ``LayeredIgnore.is_ignored()`` plus an early-return for empty /
|
|
167
|
+
undecodable files inside each ``process_*_file``. Files that early-return
|
|
168
|
+
never tick, so this pre-walk OVERSTATES the total by the ignored / empty
|
|
169
|
+
count. The parent clamps the bar to 100% on the terminal ``status=done``
|
|
170
|
+
event, so the over-count cannot stall the bar.
|
|
171
|
+
|
|
172
|
+
Mirrors the three ``localfs.walk_dir`` matchers in ``app_main``:
|
|
173
|
+
- ``**/*.java``
|
|
174
|
+
- ``**/src/main/resources/db/migration/*.sql``
|
|
175
|
+
- ``**/src/main/resources/application*.yml`` and ``.yaml``
|
|
176
|
+
"""
|
|
177
|
+
ignore = LayeredIgnore(project_root)
|
|
178
|
+
excluded = ignore.cocoindex_excluded_patterns()
|
|
179
|
+
|
|
180
|
+
def _excluded(rel_posix: str) -> bool:
|
|
181
|
+
return any(fnmatch(rel_posix, pat) for pat in excluded)
|
|
182
|
+
|
|
183
|
+
total = 0
|
|
184
|
+
for dirpath, dirnames, filenames in os.walk(project_root):
|
|
185
|
+
# Prune the same universal nuisance dirs as iter_java_source_files /
|
|
186
|
+
# cocoindex walk. (build-output pruning is matcher-dependent in the
|
|
187
|
+
# real walk; for an APPROXIMATE total this cheap prune is sufficient
|
|
188
|
+
# — the clamp absorbs any residual divergence.)
|
|
189
|
+
dirnames[:] = [
|
|
190
|
+
d for d in dirnames if d not in (".git", ".hg", ".svn", "node_modules", ".venv", "venv")
|
|
191
|
+
]
|
|
192
|
+
for fn in filenames:
|
|
193
|
+
full = Path(dirpath) / fn
|
|
194
|
+
try:
|
|
195
|
+
rel = full.resolve().relative_to(project_root).as_posix()
|
|
196
|
+
except ValueError:
|
|
197
|
+
continue
|
|
198
|
+
if _excluded(rel):
|
|
199
|
+
continue
|
|
200
|
+
# Java: **/*.java
|
|
201
|
+
if fn.endswith(".java"):
|
|
202
|
+
if not ignore.is_ignored(full):
|
|
203
|
+
total += 1
|
|
204
|
+
continue
|
|
205
|
+
# SQL: **/src/main/resources/db/migration/*.sql
|
|
206
|
+
if fn.endswith(".sql") and "/db/migration/" in rel:
|
|
207
|
+
if not ignore.is_ignored(full):
|
|
208
|
+
total += 1
|
|
209
|
+
continue
|
|
210
|
+
# YAML: **/src/main/resources/application*.yml / .yaml
|
|
211
|
+
# NOTE: ``fn`` is the bare filename (e.g. ``application-cloud.yml``), so
|
|
212
|
+
# the prefix predicate must be ``fn.startswith("application")`` —
|
|
213
|
+
# ``"/application" in fn`` was always False (no leading slash in a bare
|
|
214
|
+
# name) and under-counted every application YAML, driving the pre-walk
|
|
215
|
+
# total below the actual done count. The ``rel``-based
|
|
216
|
+
# ``"/src/main/resources/"`` gate stays (full path component).
|
|
217
|
+
if fn.endswith((".yml", ".yaml")) and fn.startswith("application") and "/src/main/resources/" in rel:
|
|
218
|
+
if not ignore.is_ignored(full):
|
|
219
|
+
total += 1
|
|
220
|
+
return total
|
|
221
|
+
|
|
222
|
+
|
|
87
223
|
@dataclass
|
|
88
224
|
class JavaLanceChunk:
|
|
89
225
|
id: str
|
|
@@ -171,6 +307,40 @@ async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]
|
|
|
171
307
|
yield
|
|
172
308
|
|
|
173
309
|
|
|
310
|
+
def _parse_and_enrich_java(
|
|
311
|
+
content_bytes: bytes,
|
|
312
|
+
chunks: list[Any],
|
|
313
|
+
rel: str,
|
|
314
|
+
project_root: Path,
|
|
315
|
+
) -> list[Any]:
|
|
316
|
+
"""Parse one Java file and enrich every chunk, off the event loop.
|
|
317
|
+
|
|
318
|
+
Returns a list of :class:`graph_enrich.ChunkEnrichment` aligned 1:1 with
|
|
319
|
+
``chunks``. Intended to run via ``asyncio.to_thread`` from
|
|
320
|
+
``process_java_file`` (vectors perf lever #2): while the worker thread
|
|
321
|
+
parses + enriches, the event loop is free to drive other files and keep the
|
|
322
|
+
embedder's batching queue fed.
|
|
323
|
+
|
|
324
|
+
Thread-safety: ``parse_java`` uses a per-thread tree-sitter ``Parser``
|
|
325
|
+
(see ``ast_java._parser``), so it is safe to call concurrently from these
|
|
326
|
+
worker threads — including the transitive ``parse_java`` that ``enrich_chunk``
|
|
327
|
+
triggers via ``collect_annotation_meta_chain`` → ``_collect_annotation_decl_index``.
|
|
328
|
+
``enrich_chunk`` is otherwise pure-Python over the now-immutable AST; its
|
|
329
|
+
``lru_cache`` reads are thread-safe under the GIL.
|
|
330
|
+
"""
|
|
331
|
+
ast = parse_java(content_bytes)
|
|
332
|
+
return [
|
|
333
|
+
enrich_chunk(
|
|
334
|
+
ast,
|
|
335
|
+
chunk_start_byte=ch.start.byte_offset,
|
|
336
|
+
chunk_end_byte=ch.end.byte_offset,
|
|
337
|
+
file_path=rel,
|
|
338
|
+
project_root=project_root,
|
|
339
|
+
)
|
|
340
|
+
for ch in chunks
|
|
341
|
+
]
|
|
342
|
+
|
|
343
|
+
|
|
174
344
|
@coco.fn(memo=True)
|
|
175
345
|
async def process_java_file(
|
|
176
346
|
file: localfs.File,
|
|
@@ -178,7 +348,7 @@ async def process_java_file(
|
|
|
178
348
|
) -> None:
|
|
179
349
|
embedder = coco.use_context(EMBEDDER)
|
|
180
350
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
181
|
-
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve())
|
|
351
|
+
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve()):
|
|
182
352
|
return
|
|
183
353
|
try:
|
|
184
354
|
content = await file.read_text()
|
|
@@ -187,8 +357,13 @@ async def process_java_file(
|
|
|
187
357
|
if not content.strip():
|
|
188
358
|
return
|
|
189
359
|
|
|
360
|
+
_tick_vectors_done()
|
|
361
|
+
|
|
190
362
|
language = detect_code_language(filename=file.file_path.path.name) or "text"
|
|
191
363
|
cs, mn, ov = JAVA_CHUNK
|
|
364
|
+
# ``splitter.split`` stays inline: the module-level ``RecursiveSplitter``
|
|
365
|
+
# shares one Rust object, so keeping split on the event loop preserves its
|
|
366
|
+
# existing single-threaded access (no new cross-file concurrency hazard).
|
|
192
367
|
chunks = splitter.split(
|
|
193
368
|
content,
|
|
194
369
|
cs,
|
|
@@ -198,18 +373,21 @@ async def process_java_file(
|
|
|
198
373
|
)
|
|
199
374
|
rel = file.file_path.path.as_posix()
|
|
200
375
|
content_bytes = content.encode("utf-8", errors="replace")
|
|
201
|
-
ast = parse_java(content_bytes)
|
|
202
376
|
|
|
203
|
-
|
|
377
|
+
# (vectors perf lever #2) parse + enrich off the event loop so the loop can
|
|
378
|
+
# keep the embedder's batching queue fed while this file is being parsed.
|
|
379
|
+
# parse_java is thread-safe (per-thread tree-sitter Parser in ast_java).
|
|
380
|
+
enrichments = await asyncio.to_thread(
|
|
381
|
+
_parse_and_enrich_java, content_bytes, chunks, rel, project_root
|
|
382
|
+
)
|
|
383
|
+
# (vectors perf lever #1) embed all chunks concurrently so the batched
|
|
384
|
+
# embedder groups them into one ``model.encode(...)`` (max_batch_size=64)
|
|
385
|
+
# instead of N serial batch-of-1 calls. Dominant win for ``increment``
|
|
386
|
+
# (few changed files → little cross-file concurrency → otherwise no batching).
|
|
387
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
388
|
+
|
|
389
|
+
for ch, enrich, emb in zip(chunks, enrichments, embeddings):
|
|
204
390
|
rs, re = chunk_key_range(ch)
|
|
205
|
-
enrich = enrich_chunk(
|
|
206
|
-
ast,
|
|
207
|
-
chunk_start_byte=ch.start.byte_offset,
|
|
208
|
-
chunk_end_byte=ch.end.byte_offset,
|
|
209
|
-
file_path=rel,
|
|
210
|
-
project_root=project_root,
|
|
211
|
-
)
|
|
212
|
-
emb = await embedder.embed(ch.text)
|
|
213
391
|
table.declare_row(
|
|
214
392
|
row=JavaLanceChunk(
|
|
215
393
|
id=str(uuid.uuid4()),
|
|
@@ -242,7 +420,7 @@ async def process_sql_file(
|
|
|
242
420
|
) -> None:
|
|
243
421
|
embedder = coco.use_context(EMBEDDER)
|
|
244
422
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
245
|
-
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve())
|
|
423
|
+
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve()):
|
|
246
424
|
return
|
|
247
425
|
try:
|
|
248
426
|
content = await file.read_text()
|
|
@@ -251,6 +429,8 @@ async def process_sql_file(
|
|
|
251
429
|
if not content.strip():
|
|
252
430
|
return
|
|
253
431
|
|
|
432
|
+
_tick_vectors_done()
|
|
433
|
+
|
|
254
434
|
language = "sql"
|
|
255
435
|
cs, mn, ov = SQL_CHUNK
|
|
256
436
|
chunks = splitter.split(
|
|
@@ -262,9 +442,11 @@ async def process_sql_file(
|
|
|
262
442
|
)
|
|
263
443
|
rel = file.file_path.path.as_posix()
|
|
264
444
|
|
|
265
|
-
|
|
445
|
+
# (vectors perf lever #1) embed chunks concurrently → batched encode.
|
|
446
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
447
|
+
|
|
448
|
+
for ch, emb in zip(chunks, embeddings):
|
|
266
449
|
rs, re = chunk_key_range(ch)
|
|
267
|
-
emb = await embedder.embed(ch.text)
|
|
268
450
|
table.declare_row(
|
|
269
451
|
row=SqlLanceChunk(
|
|
270
452
|
id=str(uuid.uuid4()),
|
|
@@ -286,7 +468,7 @@ async def process_yaml_file(
|
|
|
286
468
|
) -> None:
|
|
287
469
|
embedder = coco.use_context(EMBEDDER)
|
|
288
470
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
289
|
-
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve())
|
|
471
|
+
if LayeredIgnore(project_root).is_ignored((project_root / file.file_path.path).resolve()):
|
|
290
472
|
return
|
|
291
473
|
try:
|
|
292
474
|
content = await file.read_text()
|
|
@@ -295,6 +477,8 @@ async def process_yaml_file(
|
|
|
295
477
|
if not content.strip():
|
|
296
478
|
return
|
|
297
479
|
|
|
480
|
+
_tick_vectors_done()
|
|
481
|
+
|
|
298
482
|
ext = file.file_path.path.suffix.lower()
|
|
299
483
|
language = "yaml" if ext in (".yml", ".yaml") else "text"
|
|
300
484
|
cs, mn, ov = YAML_CHUNK
|
|
@@ -307,9 +491,11 @@ async def process_yaml_file(
|
|
|
307
491
|
)
|
|
308
492
|
rel = file.file_path.path.as_posix()
|
|
309
493
|
|
|
310
|
-
|
|
494
|
+
# (vectors perf lever #1) embed chunks concurrently → batched encode.
|
|
495
|
+
embeddings = await asyncio.gather(*(embedder.embed(ch.text) for ch in chunks))
|
|
496
|
+
|
|
497
|
+
for ch, emb in zip(chunks, embeddings):
|
|
311
498
|
rs, re = chunk_key_range(ch)
|
|
312
|
-
emb = await embedder.embed(ch.text)
|
|
313
499
|
table.declare_row(
|
|
314
500
|
row=YamlLanceChunk(
|
|
315
501
|
id=str(uuid.uuid4()),
|
|
@@ -360,8 +546,42 @@ async def app_main() -> None:
|
|
|
360
546
|
)
|
|
361
547
|
|
|
362
548
|
project_root = coco.use_context(PROJECT_ROOT)
|
|
549
|
+
# Warm per-project enrichment caches ONCE on the event-loop thread, BEFORE
|
|
550
|
+
# coco.mount_each fans files into worker threads. collect_annotation_meta_chain
|
|
551
|
+
# and load_brownfield_overrides are lru_cached per (resolved) project root;
|
|
552
|
+
# without warming, the first wave of concurrent process_java_file worker
|
|
553
|
+
# threads each cold-miss and redundantly walk+parse the ENTIRE project (a
|
|
554
|
+
# thundering herd that would offset the embedding-batching win on large
|
|
555
|
+
# repos — perf lever #2 made enrich concurrent). With warming, every worker
|
|
556
|
+
# hits a populated cache (lru_cache reads are thread-safe). Key derivation
|
|
557
|
+
# mirrors enrich_chunk exactly so the warmed entries are the ones workers hit.
|
|
558
|
+
try:
|
|
559
|
+
load_brownfield_overrides(project_root)
|
|
560
|
+
try:
|
|
561
|
+
prs = str(Path(project_root).resolve())
|
|
562
|
+
except OSError:
|
|
563
|
+
prs = str(project_root)
|
|
564
|
+
collect_annotation_meta_chain(prs)
|
|
565
|
+
except Exception:
|
|
566
|
+
# Warm-up must never break indexing — a failure just means workers
|
|
567
|
+
# cold-miss lazily (the pre-warming behavior). Swallow and continue.
|
|
568
|
+
pass
|
|
363
569
|
_ignore = LayeredIgnore(project_root)
|
|
364
570
|
_walk_excludes = _ignore.cocoindex_excluded_patterns()
|
|
571
|
+
# Emit ONE approximate total so the parent's renderer can show a determinate
|
|
572
|
+
# bar (clamps to 100% on the terminal vectors event the parent emits on
|
|
573
|
+
# cocoindex exit). Approximate — ignored / empty files over-state it; see
|
|
574
|
+
# ``_approximate_vectors_total``. ``--full-reprocess`` only: on incremental
|
|
575
|
+
# catch-up the @coco.fn(memo=True) cache skips unchanged files, so no total
|
|
576
|
+
# is knowable up front → the parent renders indeterminate from the absence.
|
|
577
|
+
try:
|
|
578
|
+
total = _approximate_vectors_total(project_root)
|
|
579
|
+
if total > 0:
|
|
580
|
+
_emit_vectors_progress(total=total, status="running")
|
|
581
|
+
except Exception:
|
|
582
|
+
# The pre-walk must never break indexing — a failure here just means
|
|
583
|
+
# the parent falls back to indeterminate. Swallow and continue.
|
|
584
|
+
pass
|
|
365
585
|
java_files = localfs.walk_dir(
|
|
366
586
|
PROJECT_ROOT,
|
|
367
587
|
recursive=True,
|
ladybug_queries.py
CHANGED
|
@@ -1161,8 +1161,11 @@ class LadybugGraph:
|
|
|
1161
1161
|
)
|
|
1162
1162
|
return [str(r["id"]) for r in rows2 if r.get("id")]
|
|
1163
1163
|
|
|
1164
|
-
def
|
|
1165
|
-
self,
|
|
1164
|
+
def _walk_calls(
|
|
1165
|
+
self,
|
|
1166
|
+
needle: str,
|
|
1167
|
+
*,
|
|
1168
|
+
side: str,
|
|
1166
1169
|
depth: int = 1,
|
|
1167
1170
|
limit: int = 100,
|
|
1168
1171
|
min_confidence: float = 0.0,
|
|
@@ -1170,6 +1173,17 @@ class LadybugGraph:
|
|
|
1170
1173
|
module: str | None = None,
|
|
1171
1174
|
microservice: str | None = None,
|
|
1172
1175
|
) -> list[CallEdge]:
|
|
1176
|
+
"""BFS the CALLS graph outward from ``needle`` along one relationship end.
|
|
1177
|
+
|
|
1178
|
+
``side="callers"`` treats the needle as the callee: the frontier matches
|
|
1179
|
+
the ``callee`` end and discovered/expanded/external-filtered nodes are the
|
|
1180
|
+
``caller`` (src) end. ``side="callees"`` is the mirror. The two public
|
|
1181
|
+
methods differ only in that orientation, so the BFS body is shared here.
|
|
1182
|
+
"""
|
|
1183
|
+
if side == "callers":
|
|
1184
|
+
scope_alias, frontier_end, discovered = "caller", "callee", "src"
|
|
1185
|
+
else:
|
|
1186
|
+
scope_alias, frontier_end, discovered = "callee", "caller", "dst"
|
|
1173
1187
|
frontier = self._method_ids_for_call_graph_needle(needle, limit=max(limit, 50))
|
|
1174
1188
|
if not frontier:
|
|
1175
1189
|
return []
|
|
@@ -1182,8 +1196,8 @@ class LadybugGraph:
|
|
|
1182
1196
|
"frontier": list(frontier),
|
|
1183
1197
|
"minc": float(min_confidence),
|
|
1184
1198
|
}
|
|
1185
|
-
sc = _scope_filters(
|
|
1186
|
-
wh_parts = ["
|
|
1199
|
+
sc = _scope_filters(scope_alias, module=module, microservice=microservice, params=params)
|
|
1200
|
+
wh_parts = [f"{frontier_end}.id IN $frontier", "c.confidence >= $minc"]
|
|
1187
1201
|
wh_parts.extend(sc)
|
|
1188
1202
|
wh = " AND ".join(wh_parts)
|
|
1189
1203
|
q = (
|
|
@@ -1197,16 +1211,19 @@ class LadybugGraph:
|
|
|
1197
1211
|
next_frontier: list[str] = []
|
|
1198
1212
|
for row in self._rows(q, params):
|
|
1199
1213
|
ce = _row_to_call_edge(row)
|
|
1200
|
-
#
|
|
1201
|
-
#
|
|
1202
|
-
|
|
1214
|
+
# The needle itself may be external (e.g. java.util.List#add);
|
|
1215
|
+
# filter only the discovered end so internal callers/callees
|
|
1216
|
+
# that touch it are still surfaced.
|
|
1217
|
+
disc_fqn = ce.src.fqn if discovered == "src" else ce.dst.fqn
|
|
1218
|
+
disc_id = ce.src.id if discovered == "src" else ce.dst.id
|
|
1219
|
+
if exclude_external and _is_external_fqn(disc_fqn):
|
|
1203
1220
|
continue
|
|
1204
1221
|
key = (ce.src.id, ce.dst.id, ce.call_site_line, ce.call_site_byte)
|
|
1205
1222
|
if key in seen:
|
|
1206
1223
|
continue
|
|
1207
1224
|
seen.add(key)
|
|
1208
1225
|
out.append(ce)
|
|
1209
|
-
next_frontier.append(
|
|
1226
|
+
next_frontier.append(disc_id)
|
|
1210
1227
|
if len(out) >= limit:
|
|
1211
1228
|
return out
|
|
1212
1229
|
frontier = list(dict.fromkeys(next_frontier))
|
|
@@ -1214,6 +1231,21 @@ class LadybugGraph:
|
|
|
1214
1231
|
break
|
|
1215
1232
|
return out
|
|
1216
1233
|
|
|
1234
|
+
def find_callers(
|
|
1235
|
+
self, needle: str, *,
|
|
1236
|
+
depth: int = 1,
|
|
1237
|
+
limit: int = 100,
|
|
1238
|
+
min_confidence: float = 0.0,
|
|
1239
|
+
exclude_external: bool = True,
|
|
1240
|
+
module: str | None = None,
|
|
1241
|
+
microservice: str | None = None,
|
|
1242
|
+
) -> list[CallEdge]:
|
|
1243
|
+
return self._walk_calls(
|
|
1244
|
+
needle, side="callers", depth=depth, limit=limit,
|
|
1245
|
+
min_confidence=min_confidence, exclude_external=exclude_external,
|
|
1246
|
+
module=module, microservice=microservice,
|
|
1247
|
+
)
|
|
1248
|
+
|
|
1217
1249
|
def find_callees(
|
|
1218
1250
|
self, needle: str, *,
|
|
1219
1251
|
depth: int = 1,
|
|
@@ -1223,49 +1255,11 @@ class LadybugGraph:
|
|
|
1223
1255
|
module: str | None = None,
|
|
1224
1256
|
microservice: str | None = None,
|
|
1225
1257
|
) -> list[CallEdge]:
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
out: list[CallEdge] = []
|
|
1232
|
-
seen: set[tuple[str, str, int, int]] = set()
|
|
1233
|
-
for _ in range(max(1, int(depth))):
|
|
1234
|
-
params: dict[str, Any] = {
|
|
1235
|
-
"frontier": list(frontier),
|
|
1236
|
-
"minc": float(min_confidence),
|
|
1237
|
-
}
|
|
1238
|
-
sc = _scope_filters("callee", module=module, microservice=microservice, params=params)
|
|
1239
|
-
wh_parts = ["caller.id IN $frontier", "c.confidence >= $minc"]
|
|
1240
|
-
wh_parts.extend(sc)
|
|
1241
|
-
wh = " AND ".join(wh_parts)
|
|
1242
|
-
q = (
|
|
1243
|
-
f"MATCH (caller:Symbol)-[c:CALLS]->(callee:Symbol) WHERE {wh} "
|
|
1244
|
-
f"RETURN {caller_proj}, {callee_proj}, "
|
|
1245
|
-
f"c.call_site_line AS call_site_line, c.call_site_byte AS call_site_byte, "
|
|
1246
|
-
f"c.arg_count AS arg_count, c.confidence AS confidence, c.strategy AS strategy, "
|
|
1247
|
-
f"c.source AS source, c.resolved AS resolved "
|
|
1248
|
-
f"LIMIT {int(limit) * 8}"
|
|
1249
|
-
)
|
|
1250
|
-
next_frontier: list[str] = []
|
|
1251
|
-
for row in self._rows(q, params):
|
|
1252
|
-
ce = _row_to_call_edge(row)
|
|
1253
|
-
# Filter only discovered callees (dst). Needle may be external while
|
|
1254
|
-
# still listing non-external outbound calls when any exist.
|
|
1255
|
-
if exclude_external and _is_external_fqn(ce.dst.fqn):
|
|
1256
|
-
continue
|
|
1257
|
-
key = (ce.src.id, ce.dst.id, ce.call_site_line, ce.call_site_byte)
|
|
1258
|
-
if key in seen:
|
|
1259
|
-
continue
|
|
1260
|
-
seen.add(key)
|
|
1261
|
-
out.append(ce)
|
|
1262
|
-
next_frontier.append(ce.dst.id)
|
|
1263
|
-
if len(out) >= limit:
|
|
1264
|
-
return out
|
|
1265
|
-
frontier = list(dict.fromkeys(next_frontier))
|
|
1266
|
-
if not frontier:
|
|
1267
|
-
break
|
|
1268
|
-
return out
|
|
1258
|
+
return self._walk_calls(
|
|
1259
|
+
needle, side="callees", depth=depth, limit=limit,
|
|
1260
|
+
min_confidence=min_confidence, exclude_external=exclude_external,
|
|
1261
|
+
module=module, microservice=microservice,
|
|
1262
|
+
)
|
|
1269
1263
|
|
|
1270
1264
|
def expand_methods(
|
|
1271
1265
|
self, fqns: list[str], *, depth: int = 1,
|
mcp_v2.py
CHANGED
|
@@ -817,7 +817,7 @@ def _merge_overrides_edge_summary(
|
|
|
817
817
|
"""Reconcile `OVERRIDES` with `override_axis_rollup_for` without clobbering stored `in`.
|
|
818
818
|
|
|
819
819
|
Rollup rows reuse the ``OVERRIDES`` key for dispatch-up counts only (``in`` is always
|
|
820
|
-
zero there). Stored ``[:OVERRIDES]`` edges contribute real ``in``/``out`` from
|
|
820
|
+
zero there). Stored ``[:OVERRIDES]`` edges contribute real ``in``/``out`` from LadybugDB;
|
|
821
821
|
merge per direction with ``max`` so inbound override edges stay visible.
|
|
822
822
|
"""
|
|
823
823
|
roll = _incident_counts(summary_after_rollups.get("OVERRIDES"))
|
|
@@ -1095,9 +1095,9 @@ def describe_v2(
|
|
|
1095
1095
|
has_id = bool(id and str(id).strip())
|
|
1096
1096
|
has_fqn = bool(fqn and str(fqn).strip())
|
|
1097
1097
|
if not has_id and not has_fqn:
|
|
1098
|
-
return DescribeOutput(success=False, message="id or fqn required"
|
|
1098
|
+
return DescribeOutput(success=False, message="id or fqn required")
|
|
1099
1099
|
if has_id and str(id).strip().startswith("ucs:"):
|
|
1100
|
-
return DescribeOutput(success=False, message=_DESCRIBE_UCS_ID_MESSAGE
|
|
1100
|
+
return DescribeOutput(success=False, message=_DESCRIBE_UCS_ID_MESSAGE)
|
|
1101
1101
|
hint_message: str | None = None
|
|
1102
1102
|
node_id: str
|
|
1103
1103
|
if has_id:
|
|
@@ -1109,7 +1109,7 @@ def describe_v2(
|
|
|
1109
1109
|
{"fqn": fqn_val},
|
|
1110
1110
|
)
|
|
1111
1111
|
if not rows:
|
|
1112
|
-
return DescribeOutput(success=False, message=f"No Symbol found for fqn='{fqn_val}'"
|
|
1112
|
+
return DescribeOutput(success=False, message=f"No Symbol found for fqn='{fqn_val}'")
|
|
1113
1113
|
node_id = str(rows[0]["id"] or "")
|
|
1114
1114
|
if len(rows) > 1:
|
|
1115
1115
|
hint_message = (
|
|
@@ -1784,7 +1784,7 @@ def neighbors_v2(
|
|
|
1784
1784
|
)
|
|
1785
1785
|
except ValueError as exc:
|
|
1786
1786
|
_log_fail_loud("edge_filter")
|
|
1787
|
-
return NeighborsOutput(success=False, message=str(exc),
|
|
1787
|
+
return NeighborsOutput(success=False, message=str(exc), requested_edge_types=[])
|
|
1788
1788
|
if include_unresolved and ef is not None:
|
|
1789
1789
|
return NeighborsOutput(
|
|
1790
1790
|
success=False,
|
|
@@ -1792,21 +1792,18 @@ def neighbors_v2(
|
|
|
1792
1792
|
"include_unresolved=True is incompatible with edge_filter; "
|
|
1793
1793
|
"UnresolvedCallSite rows have no edge attributes to filter on"
|
|
1794
1794
|
),
|
|
1795
|
-
hints=[],
|
|
1796
1795
|
requested_edge_types=requested_edge_types,
|
|
1797
1796
|
)
|
|
1798
1797
|
if include_unresolved and requested_edge_types != ["CALLS"]:
|
|
1799
1798
|
return NeighborsOutput(
|
|
1800
1799
|
success=False,
|
|
1801
1800
|
message="include_unresolved requires edge_types=['CALLS']",
|
|
1802
|
-
hints=[],
|
|
1803
1801
|
requested_edge_types=requested_edge_types,
|
|
1804
1802
|
)
|
|
1805
1803
|
if include_unresolved and direction != "out":
|
|
1806
1804
|
return NeighborsOutput(
|
|
1807
1805
|
success=False,
|
|
1808
1806
|
message='include_unresolved requires direction="out"',
|
|
1809
|
-
hints=[],
|
|
1810
1807
|
requested_edge_types=requested_edge_types,
|
|
1811
1808
|
)
|
|
1812
1809
|
if ef and (err := _edgefilter_applicability_error(requested_edge_types, ef)):
|
|
@@ -1814,17 +1811,15 @@ def neighbors_v2(
|
|
|
1814
1811
|
return NeighborsOutput(
|
|
1815
1812
|
success=False,
|
|
1816
1813
|
message=err,
|
|
1817
|
-
hints=[],
|
|
1818
1814
|
requested_edge_types=requested_edge_types,
|
|
1819
1815
|
)
|
|
1820
1816
|
if nf and (err := _validate_no_wildcards(nf)):
|
|
1821
1817
|
_log_fail_loud("wildcard")
|
|
1822
|
-
return NeighborsOutput(success=False, message=err,
|
|
1818
|
+
return NeighborsOutput(success=False, message=err, requested_edge_types=[])
|
|
1823
1819
|
if composed_keys and direction != "out":
|
|
1824
1820
|
return NeighborsOutput(
|
|
1825
1821
|
success=False,
|
|
1826
1822
|
message='Composed edge types require direction="out"',
|
|
1827
|
-
hints=[],
|
|
1828
1823
|
requested_edge_types=requested_edge_types,
|
|
1829
1824
|
)
|
|
1830
1825
|
use_calls_path = flat_labels == ["CALLS"] and not composed_keys
|
|
@@ -1849,7 +1844,6 @@ def neighbors_v2(
|
|
|
1849
1844
|
return NeighborsOutput(
|
|
1850
1845
|
success=False,
|
|
1851
1846
|
message=axis_msg,
|
|
1852
|
-
hints=[],
|
|
1853
1847
|
requested_edge_types=requested_edge_types,
|
|
1854
1848
|
)
|
|
1855
1849
|
origin_row = _load_node_record(g, origin_id, "symbol")
|
|
@@ -1865,7 +1859,6 @@ def neighbors_v2(
|
|
|
1865
1859
|
return NeighborsOutput(
|
|
1866
1860
|
success=False,
|
|
1867
1861
|
message=err,
|
|
1868
|
-
hints=[],
|
|
1869
1862
|
requested_edge_types=requested_edge_types,
|
|
1870
1863
|
)
|
|
1871
1864
|
if use_calls_path:
|
|
@@ -1891,7 +1884,6 @@ def neighbors_v2(
|
|
|
1891
1884
|
return NeighborsOutput(
|
|
1892
1885
|
success=False,
|
|
1893
1886
|
message=str(exc),
|
|
1894
|
-
hints=[],
|
|
1895
1887
|
requested_edge_types=requested_edge_types,
|
|
1896
1888
|
)
|
|
1897
1889
|
if (
|
|
@@ -1904,7 +1896,7 @@ def neighbors_v2(
|
|
|
1904
1896
|
results.extend(origin_edges)
|
|
1905
1897
|
continue
|
|
1906
1898
|
if flat_labels:
|
|
1907
|
-
#
|
|
1899
|
+
# Some Cypher binders can drop `label(e) IN $list` in WHERE; use OR of scalar equalities.
|
|
1908
1900
|
label_params = [f"l{i}" for i in range(len(flat_labels))]
|
|
1909
1901
|
label_predicate = "(" + " OR ".join(f"label(e) = ${name}" for name in label_params) + ")"
|
|
1910
1902
|
q_params = {"id": origin_id, **dict(zip(label_params, flat_labels, strict=True))}
|
|
@@ -1941,7 +1933,7 @@ def neighbors_v2(
|
|
|
1941
1933
|
if nf and (err := _nodefilter_applicability_error(other_kind, nf)):
|
|
1942
1934
|
_log_fail_loud("applicability")
|
|
1943
1935
|
return NeighborsOutput(
|
|
1944
|
-
success=False, message=err,
|
|
1936
|
+
success=False, message=err, requested_edge_types=[]
|
|
1945
1937
|
)
|
|
1946
1938
|
if not _node_matches_filter(other_kind, other_rec, nf):
|
|
1947
1939
|
continue
|
|
@@ -1968,7 +1960,7 @@ def neighbors_v2(
|
|
|
1968
1960
|
if nf and (err := _nodefilter_applicability_error(other_kind, nf)):
|
|
1969
1961
|
_log_fail_loud("applicability")
|
|
1970
1962
|
return NeighborsOutput(
|
|
1971
|
-
success=False, message=err,
|
|
1963
|
+
success=False, message=err, requested_edge_types=[]
|
|
1972
1964
|
)
|
|
1973
1965
|
if not _node_matches_filter(other_kind, other_rec, nf):
|
|
1974
1966
|
continue
|