java-codebase-rag 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl

java_index_flow_lancedb.py

@@ -18,10 +18,13 @@ from __future__ import annotations
 
 import inspect
 import os
+import sys
+import threading
 import uuid
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
+from fnmatch import fnmatch
 from pathlib import Path
 from typing import Annotated, Any
 
@@ -84,6 +87,138 @@ splitter = RecursiveSplitter()
 _NUM_TXN_BEFORE_OPTIMIZE = 10**12
 
 
+# --- Vectors-phase progress emission (JCIRAG_PROGRESS kind=vectors) -----------
+#
+# The flow runs in a CHILD cocoindex process; it prints structured progress to
+# its stderr and the parent (pipeline._popen_capturing_stderr /
+# cli_progress.accumulate_and_relay_subprocess_streams) parses it via
+# ProgressRelay and feeds the renderer. The flow CANNOT know when all files are
+# done (cocoindex offers no "all files done" hook in the flow), so it emits:
+#   - ONE ``total=N status=running`` line from ``app_main`` (approximate
+#     pre-walk: matcher includes + LayeredIgnore), and
+#   - per-file ``done=k status=running`` ticks (throttled every ~25 files) from
+#     ``process_*_file`` (shared atomic counter).
+# The PARENT emits the terminal ``status=done``/``failed`` vectors event on
+# cocoindex exit (drives clamp-on-completion + phase transition to Optimize).
+
+# Per-file tick cadence: bound stderr volume on huge trees without making the
+# bar feel stale. Every 25th file (and the modulo boundary is enough — the
+# parent clamps to total on the terminal event anyway).
+_VECTORS_TICK_EVERY = 25
+
+# Thread-safe counter: cocoindex may call process_*_file concurrently
+# (mount_each parallelism is implementation-defined). A module-level lock guards
+# both the counter and the emission so two threads never interleave a tick.
+_vectors_done_lock = threading.Lock()
+_vectors_done_count = 0
+
+
+def _emit_vectors_progress(
+    *,
+    done: int | None = None,
+    total: int | None = None,
+    status: str = "running",
+    elapsed_s: float | None = None,
+) -> None:
+    """Emit one ``JCIRAG_PROGRESS kind=vectors …`` line to stderr (flushed).
+
+    Field order is fixed (kind, done, total, status, elapsed_s) so the parser
+    and tests can pin substrings. Omitted fields are simply absent.
+    """
+    fields = ["kind=vectors"]
+    if done is not None:
+        fields.append(f"done={done}")
+    if total is not None:
+        fields.append(f"total={total}")
+    fields.append(f"status={status}")
+    if elapsed_s is not None:
+        fields.append(f"elapsed_s={elapsed_s:.2f}")
+    print("JCIRAG_PROGRESS " + " ".join(fields), file=sys.stderr, flush=True)
+
+
+def _tick_vectors_done() -> None:
+    """Increment the shared per-file counter and emit a throttled ``done=k`` tick.
+
+    Called once per successfully-processed file (after the ignore / empty
+    early-returns). The tick is emitted every ``_VECTORS_TICK_EVERY`` files so
+    stderr volume stays bounded on huge trees; the parent clamps to total on
+    the terminal event, so the exact tick cadence is not load-bearing.
+    """
+    global _vectors_done_count
+    with _vectors_done_lock:
+        _vectors_done_count += 1
+        n = _vectors_done_count
+        if n % _VECTORS_TICK_EVERY != 0:
+            return
+        # Emit under the lock: the docstring above promises the lock guards both
+        # the counter AND the emission, so two concurrent ticks can't emit their
+        # ``done=N`` lines out of order. Contention is negligible (fires every
+        # ~25 files).
+        _emit_vectors_progress(done=n, status="running")
+
+
+def _approximate_vectors_total(project_root: Path) -> int:
+    """Reproduce the matchers' include globs + LayeredIgnore for an approximate total.
+
+    The flow applies two filtering layers: (1) ``PatternFilePathMatcher``
+    excludes at walk time via ``LayeredIgnore.cocoindex_excluded_patterns()``,
+    then (2) ``LayeredIgnore.is_ignored()`` plus an early-return for empty /
+    undecodable files inside each ``process_*_file``. Files that early-return
+    never tick, so this pre-walk OVERSTATES the total by the ignored / empty
+    count. The parent clamps the bar to 100% on the terminal ``status=done``
+    event, so the over-count cannot stall the bar.
+
+    Mirrors the three ``localfs.walk_dir`` matchers in ``app_main``:
+      - ``**/*.java``
+      - ``**/src/main/resources/db/migration/*.sql``
+      - ``**/src/main/resources/application*.yml`` and ``.yaml``
+    """
+    ignore = LayeredIgnore(project_root)
+    excluded = ignore.cocoindex_excluded_patterns()
+
+    def _excluded(rel_posix: str) -> bool:
+        return any(fnmatch(rel_posix, pat) for pat in excluded)
+
+    total = 0
+    for dirpath, dirnames, filenames in os.walk(project_root):
+        # Prune the same universal nuisance dirs as iter_java_source_files /
+        # cocoindex walk. (build-output pruning is matcher-dependent in the
+        # real walk; for an APPROXIMATE total this cheap prune is sufficient
+        # — the clamp absorbs any residual divergence.)
+        dirnames[:] = [
+            d for d in dirnames if d not in (".git", ".hg", ".svn", "node_modules", ".venv", "venv")
+        ]
+        for fn in filenames:
+            full = Path(dirpath) / fn
+            try:
+                rel = full.resolve().relative_to(project_root).as_posix()
+            except ValueError:
+                continue
+            if _excluded(rel):
+                continue
+            # Java: **/*.java
+            if fn.endswith(".java"):
+                if not ignore.is_ignored(full)[0]:
+                    total += 1
+                continue
+            # SQL: **/src/main/resources/db/migration/*.sql
+            if fn.endswith(".sql") and "/db/migration/" in rel:
+                if not ignore.is_ignored(full)[0]:
+                    total += 1
+                continue
+            # YAML: **/src/main/resources/application*.yml / .yaml
+            # NOTE: ``fn`` is the bare filename (e.g. ``application-cloud.yml``), so
+            # the prefix predicate must be ``fn.startswith("application")`` —
+            # ``"/application" in fn`` was always False (no leading slash in a bare
+            # name) and under-counted every application YAML, driving the pre-walk
+            # total below the actual done count. The ``rel``-based
+            # ``"/src/main/resources/"`` gate stays (full path component).
+            if fn.endswith((".yml", ".yaml")) and fn.startswith("application") and "/src/main/resources/" in rel:
+                if not ignore.is_ignored(full)[0]:
+                    total += 1
+    return total
+
+
 @dataclass
 class JavaLanceChunk:
     id: str
@@ -187,6 +322,8 @@ async def process_java_file(
     if not content.strip():
         return
 
+    _tick_vectors_done()
+
     language = detect_code_language(filename=file.file_path.path.name) or "text"
     cs, mn, ov = JAVA_CHUNK
     chunks = splitter.split(
@@ -251,6 +388,8 @@ async def process_sql_file(
     if not content.strip():
         return
 
+    _tick_vectors_done()
+
     language = "sql"
     cs, mn, ov = SQL_CHUNK
     chunks = splitter.split(
@@ -295,6 +434,8 @@ async def process_yaml_file(
     if not content.strip():
         return
 
+    _tick_vectors_done()
+
     ext = file.file_path.path.suffix.lower()
     language = "yaml" if ext in (".yml", ".yaml") else "text"
     cs, mn, ov = YAML_CHUNK
@@ -362,6 +503,20 @@ async def app_main() -> None:
     project_root = coco.use_context(PROJECT_ROOT)
     _ignore = LayeredIgnore(project_root)
     _walk_excludes = _ignore.cocoindex_excluded_patterns()
+    # Emit ONE approximate total so the parent's renderer can show a determinate
+    # bar (clamps to 100% on the terminal vectors event the parent emits on
+    # cocoindex exit). Approximate — ignored / empty files over-state it; see
+    # ``_approximate_vectors_total``. ``--full-reprocess`` only: on incremental
+    # catch-up the @coco.fn(memo=True) cache skips unchanged files, so no total
+    # is knowable up front → the parent renders indeterminate from the absence.
+    try:
+        total = _approximate_vectors_total(project_root)
+        if total > 0:
+            _emit_vectors_progress(total=total, status="running")
+    except Exception:
+        # The pre-walk must never break indexing — a failure here just means
+        # the parent falls back to indeterminate. Swallow and continue.
+        pass
     java_files = localfs.walk_dir(
         PROJECT_ROOT,
         recursive=True,

server.py

@@ -13,9 +13,8 @@ import mcp_v2
 from index_common import SBERT_MODEL
 from java_codebase_rag.cli_progress import (
     accumulate_and_relay_subprocess_streams,
-    emit_vectors_finish,
-    emit_vectors_start,
 )
+from java_codebase_rag.progress import ProgressEvent
 from java_codebase_rag._fdlimit import raise_fd_limit
 from java_codebase_rag.config import (
     cocoindex_subprocess_env_defaults,
@@ -113,7 +112,44 @@ class ScopeManager:
 
     def _detect_scope(self) -> str | None:
         from graph_enrich import detect_microservice_from_path
-        return detect_microservice_from_path(Path.cwd(), self.source_root)
+
+        candidate = detect_microservice_from_path(Path.cwd(), self.source_root)
+        if candidate is None:
+            return None
+        # Only auto-scope to a microservice that actually has indexed code.
+        # detect_microservice_from_path can mislabel a non-microservice
+        # top-level child of source_root — most importantly the config/context
+        # directory the MCP server is launched from (no build marker, no
+        # source) — via its "first path segment under root" fallback. Scoping
+        # every query to such a name yields zero matches, so all tools return
+        # empty. A real microservice the operator is working in is, by
+        # definition, present in the index, so validating against the indexed
+        # set cannot suppress a legitimate scope. When the index is unreadable
+        # (empty known set) we keep the detected candidate rather than silently
+        # disabling auto-scope on a transient graph error.
+        known = self._indexed_microservices()
+        if known and candidate not in known:
+            return None
+        return candidate
+
+    def _indexed_microservices(self) -> set[str]:
+        """Microservice names that have indexed type symbols.
+
+        Graph-only source of truth: the graph is always built alongside Lance,
+        and a Lance-only index (no graph) is not a supported state. Any failure
+        (graph missing, open error, empty index) returns an empty set, which
+        ``_detect_scope`` treats as "cannot validate — keep detection".
+        """
+        try:
+            if not LadybugGraph.exists():
+                return set()
+            # LadybugGraph.get() opens the DB and runs meta(); it can raise
+            # (e.g. RuntimeError on ontology-version mismatch). Caught here ->
+            # empty set -> _detect_scope keeps the detected scope.
+            counts = LadybugGraph.get().microservice_counts()
+            return {name for name in counts if name}
+        except Exception:
+            return set()
 
     def _log_detection(self) -> None:
         if self.default_scope:
@@ -270,10 +306,20 @@ def list_code_index_tables_payload() -> IndexInfoOutput:
     )
 
 
-async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) -> RefreshIndexOutput:
+async def run_refresh_pipeline(
+    *,
+    quiet: bool = False,
+    verbose: bool = True,
+    on_progress=None,
+    on_progress_console: object | None = None,
+) -> RefreshIndexOutput:
     root = _project_root()
     cocoindex_bin = Path(sys.executable).parent / "cocoindex"
     if not cocoindex_bin.is_file():
+        # 127 pre-spawn: emit a terminal failed vectors event so the renderer's
+        # task doesn't hang at running (matches the sync pipeline path).
+        if on_progress is not None:
+            on_progress(ProgressEvent(kind="vectors", phase=None, pass_=None, done=None, total=None, status="failed", elapsed_s=None))
         return RefreshIndexOutput(
             success=False,
             message=f"cocoindex not found next to Python: {cocoindex_bin}",
@@ -286,6 +332,8 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
         if fallback.is_file():
             flow_path = fallback
         else:
+            if on_progress is not None:
+                on_progress(ProgressEvent(kind="vectors", phase=None, pass_=None, done=None, total=None, status="failed", elapsed_s=None))
             return RefreshIndexOutput(
                 success=False,
                 message=f"java_index_flow_lancedb.py not found under {root} nor {bundle_dir}",
@@ -308,13 +356,14 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
             )
             out_b, err_b = await proc.communicate()
         except Exception as exc:
+            if on_progress is not None:
+                on_progress(ProgressEvent(kind="vectors", phase=None, pass_=None, done=None, total=None, status="failed", elapsed_s=None))
             return RefreshIndexOutput(
                 success=False,
                 message=f"spawn failed: {exc!s}",
                 phases_run=[],
             )
     else:
-        emit_vectors_start()
         t0 = time.perf_counter()
         code_c = -1
         try:
@@ -329,16 +378,30 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
             )
-            out_b, err_b = await accumulate_and_relay_subprocess_streams(proc, relay=True, verbose=verbose)
+            # The vectors task is fed by the child's per-file ticks + the
+            # approximate total line, parsed by the ProgressRelay inside the
+            # async drain and routed to on_progress.
+            out_b, err_b = await accumulate_and_relay_subprocess_streams(
+                proc, relay=True, verbose=verbose,
+                on_progress=on_progress, on_progress_console=on_progress_console,
+            )
             code_c = proc.returncode if proc.returncode is not None else -1
         except Exception as exc:
+            if on_progress is not None:
+                on_progress(ProgressEvent(kind="vectors", phase=None, pass_=None, done=None, total=None, status="failed", elapsed_s=None))
             return RefreshIndexOutput(
                 success=False,
                 message=f"spawn failed: {exc!s}",
                 phases_run=[],
             )
         finally:
-            emit_vectors_finish(elapsed_s=time.perf_counter() - t0, exit_code=code_c)
+            # The parent emits the terminal vectors event (the flow can't — no
+            # "all files done" hook). Drives clamp-on-completion + phase
+            # transition to Optimize.
+            if on_progress is not None:
+                elapsed = time.perf_counter() - t0
+                status = "done" if code_c == 0 else "failed"
+                on_progress(ProgressEvent(kind="vectors", phase=None, pass_=None, done=None, total=None, status=status, elapsed_s=elapsed))
     assert proc is not None
     out = out_b.decode(errors="replace")
     err = err_b.decode(errors="replace")
@@ -366,7 +429,7 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
                 idx_dir = Path(idx_raw)
             else:
                 idx_dir = (root / ".java-codebase-rag").resolve()
-            await optimize_lance_tables(idx_dir, quiet=quiet)
+            await optimize_lance_tables(idx_dir, quiet=quiet, on_progress=on_progress)
         except Exception as exc:
             optimize_error = f"lance optimize failed: {exc}"
             print(f"java-codebase-rag: {optimize_error}", file=sys.stderr)
@@ -394,7 +457,10 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
                 if quiet:
                     gout_b, gerr_b = await gproc.communicate()
                 else:
-                    gout_b, gerr_b = await accumulate_and_relay_subprocess_streams(gproc, relay=True, verbose=verbose)
+                    gout_b, gerr_b = await accumulate_and_relay_subprocess_streams(
+                        gproc, relay=True, verbose=verbose,
+                        on_progress=on_progress, on_progress_console=on_progress_console,
+                    )
                 graph_code = gproc.returncode
                 graph_out = gout_b.decode(errors="replace")
                 graph_err = gerr_b.decode(errors="replace")