java-codebase-rag 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl

build_ast_graph.py

@@ -25,6 +25,7 @@ The LadybugDB DB is dropped and rebuilt on every run (Phase 1 is a full rebuild)
 from __future__ import annotations
 
 import argparse
+import contextlib
 import hashlib
 import json
 import logging
@@ -84,6 +85,53 @@ def _verbose_stderr_line(content: str) -> None:
         print(content, file=sys.stderr, flush=True)
 
 
+def _emit_graph_progress(parts: dict[str, object], *, verbose: bool) -> None:
+    """Emit one ``JCIRAG_PROGRESS kind=graph …`` line to stderr (gated by verbose).
+
+    The parent process (``pipeline.run_build_ast_graph`` /
+    ``run_incremental_graph``) passes ``--verbose`` in default AND verbose modes
+    (only suppressed for ``--quiet``), so this structured progress surfaces in
+    default mode (where the parent renders it) and verbose mode (raw relay). In
+    ``--quiet`` the builder is never invoked with ``--verbose`` so nothing is
+    emitted. Field order is fixed so the parser and tests can pin substrings.
+    """
+    if not verbose:
+        return
+    fields = ["kind=graph"]
+    for key in ("pass", "done", "total", "status", "elapsed_s"):
+        if key in parts:
+            fields.append(f"{key}={parts[key]}")
+    line = "JCIRAG_PROGRESS " + " ".join(fields)
+    _verbose_stderr_line(line)
+
+
+# Pass-1 per-file tick cadence: bound stderr volume on huge trees without making
+# the bar feel stale. A final tick on pass completion carries status=done.
+_PASS1_TICK_EVERY = 25
+
+
+@contextlib.contextmanager
+def _graph_pass_progress(pass_label: str, *, verbose: bool):
+    """Emit ``pass=N/6 status=running`` on entry and ``status=done elapsed_s=…``
+    on exit for passes 2–6 (each advances the rendered bar by 1/6).
+
+    Usage: ``with _graph_pass_progress("2/6", verbose=verbose): …``
+    """
+    if not verbose:
+        yield
+        return
+    _emit_graph_progress({"pass": pass_label, "status": "running"}, verbose=verbose)
+    t0 = time.time()
+    try:
+        yield
+    finally:
+        elapsed = time.time() - t0
+        _emit_graph_progress(
+            {"pass": pass_label, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
+            verbose=verbose,
+        )
+
+
 class _VerbosePassHeartbeats:
     """Emit ``[tag] running … Ns elapsed`` every 5s on stderr while in scope (verbose only)."""
 
@@ -837,7 +885,14 @@ def _register_type(
     return entry
 
 
-def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files: set[str] | None = None) -> dict[str, JavaFileAst]:
+def pass1_parse(
+    root: Path,
+    tables: GraphTables,
+    *,
+    verbose: bool,
+    scope_files: set[str] | None = None,
+    removed_files: set[str] | None = None,
+) -> dict[str, JavaFileAst]:
     """Walk files, parse them, populate node indexes. Returns path -> AST.
 
     Args:
@@ -845,6 +900,11 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
         tables: GraphTables to populate.
         verbose: Whether to emit progress output.
         scope_files: Optional set of relative POSIX paths to parse. If None, parse all files.
+        removed_files: Optional set of relative POSIX paths that no longer exist
+            on disk (incremental deletions). These are members of ``scope_files``
+            (they were deleted, so they participate in scoped deletion) but are
+            never visited by the parse walk, so they must be excluded from the
+            pass-1 total to keep ``done`` from undercounting then two-way-clamping.
     """
     asts: dict[str, JavaFileAst] = {}
     ignore = LayeredIgnore(root)
@@ -852,6 +912,23 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
     n_files = 0
     if verbose:
         _verbose_stderr_line(_PASS1_START)
+    # Count-first: one filtered walk (no parsing) to set the EXACT total before
+    # the parse loop ticks. Single-layer ignore → the count is exact, so the
+    # rendered bar is determinate. For a scoped (incremental) parse the total is
+    # the number of files that will actually be visited: scope minus any removed
+    # files (which are members of scope for deletion but gone from disk, so the
+    # parse walk never ticks them); for a full rebuild it is the non-ignored
+    # .java count.
+    if verbose:
+        if scope_files is not None:
+            removed = removed_files if removed_files is not None else set()
+            pass1_total = len(scope_files - removed)
+        else:
+            pass1_total = sum(1 for _ in iter_java_source_files(root, ignore=ignore))
+        _emit_graph_progress(
+            {"pass": "1/6", "done": 0, "total": pass1_total, "status": "running"},
+            verbose=verbose,
+        )
     slow_sec = 0.0
     raw_slow = os.environ.get("JAVA_CODEBASE_RAG_TEST_GRAPH_SLOW_SEC", "").strip()
     if raw_slow:
@@ -871,6 +948,11 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
             if scope_files is not None and rel not in scope_files:
                 continue
             n_files += 1
+            if verbose and (n_files % _PASS1_TICK_EVERY == 0):
+                _emit_graph_progress(
+                    {"pass": "1/6", "done": n_files, "status": "running"},
+                    verbose=verbose,
+                )
             try:
                 content = p.read_bytes()
             except OSError:
@@ -906,6 +988,10 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
 
     if verbose:
         elapsed = time.time() - t0
+        _emit_graph_progress(
+            {"pass": "1/6", "done": n_files, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
+            verbose=verbose,
+        )
         _verbose_stderr_line(
             f"[graph] pass 1 · parsed {n_files} files in {elapsed:.2f}s: "
             f"{len(tables.types)} types, {len(tables.members)} members, "
@@ -1145,7 +1231,7 @@ def pass2_edges(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: b
     seen_inj: set[tuple[str, str, str, str]] = set()
     if verbose:
         _verbose_stderr_line(_PASS2_START)
-    with _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
+    with _graph_pass_progress("2/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
         for fqn, entry in tables.types.items():
             ast = asts.get(entry.file_path)
             if ast is None:
@@ -1818,7 +1904,7 @@ def pass3_calls(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: b
         _verbose_stderr_line(_PASS3_START)
     _build_member_indexes(tables)
     stats = CallResolutionStats()
-    with _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
+    with _graph_pass_progress("3/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
         for rel_path, file_ast in asts.items():
             try:
                 _process_file_calls(file_ast, rel_path, tables, stats)
@@ -1972,7 +2058,7 @@ def pass4_routes(
     meta_chain = collect_annotation_meta_chain(prs)
     if verbose:
         _verbose_stderr_line(_PASS4_START)
-    with _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
+    with _graph_pass_progress("4/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
 
         for ast in asts.values():
             stats.routes_skipped_unresolved += ast.routes_skipped_unresolved
@@ -2149,7 +2235,7 @@ def pass5_imperative_edges(
 
     if verbose:
         _verbose_stderr_line(_PASS5_START)
-    with _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
+    with _graph_pass_progress("5/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
         for member in sorted(tables.members, key=lambda x: x.node_id):
             if member.decl.is_constructor:
                 continue
@@ -2551,7 +2637,7 @@ def pass6_match_edges(
 
     if verbose:
         _verbose_stderr_line(_PASS6_START)
-    with _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
+    with _graph_pass_progress("6/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
         for row in tables.http_call_rows:
             if row.match != "unresolved":
                 continue
@@ -3586,7 +3672,9 @@ def incremental_rebuild(
             _verbose_stderr_line("[increment] rebuilding scoped files (passes 1-4)")
 
         tables = GraphTables()
-        asts = pass1_parse(source_root, tables, verbose=verbose, scope_files=scope_files)
+        asts = pass1_parse(
+            source_root, tables, verbose=verbose, scope_files=scope_files, removed_files=removed
+        )
 
         # Load existing types and members for cross-file resolution (only from unchanged files)
         _load_existing_types(conn, tables, exclude_files=scope_files)

java_codebase_rag/cli.py

@@ -131,15 +131,41 @@ def _run_with_pipeline_progress(
     cfg: ResolvedOperatorConfig,
     *,
     quiet: bool,
-    work: Callable[[], int],
+    verbose: bool = False,
+    work: Callable[["PipelineProgress | None"], int],
 ) -> int:
-    if quiet:
-        return int(work())
+    """Run ``work`` under the unified progress renderer (default TTY mode only).
+
+    ``work`` receives a :class:`PipelineProgress` whose ``on_progress`` callback
+    should be forwarded to the graph/vectors pipeline helpers so their
+    ``JCIRAG_PROGRESS`` events feed the renderer. In ``--quiet`` or ``--verbose``
+    mode the context is ``None`` (no Live region: quiet is silent, verbose
+    raw-relays subprocess output).
+    """
+    if quiet or verbose:
+        return int(work(None))
+    from java_codebase_rag.progress import build_index_progress_context
+
+    # PR-3 owns all three tasks in order: Vectors → Optimize → Graph. The vectors
+    # task is fed by the cocoindex child's per-file ticks + approximate total
+    # (subprocess transport, parsed by ProgressRelay); the optimize task is fed
+    # in-process by lance_optimize; the graph task is fed by the build_ast_graph
+    # child (subprocess transport). A task only becomes visible/running once its
+    # first event arrives.
+    renderer, on_progress, console = build_index_progress_context()
+    progress = PipelineProgress(renderer=renderer)
+    progress.on_progress = on_progress
+    progress.console = console
+
     _pipeline_header(subcommand, cfg)
     t0 = time.perf_counter()
     code = 0
+    # start() always flips _started (the non-TTY fallback is a no-op for Live but
+    # still needs the flag so apply() routes to the concise-line printer). The
+    # TTY Live region is entered inside start() only when the console is a TTY.
+    renderer.start()
     try:
-        code = int(work())
+        code = int(work(progress))
         return code
     except BaseException as exc:
         # Keep footer aligned with process outcome (main maps unhandled Exception -> exit 2).
@@ -155,9 +181,26 @@ def _run_with_pipeline_progress(
             code = 2
         raise
     finally:
+        renderer.stop()
         _pipeline_footer(subcommand, t0, code)
 
 
+class PipelineProgress:
+    """Progress context handed to ``work``: the renderer + a ready ``on_progress``.
+
+    ``on_progress``/``console`` are wired by :func:`_run_with_pipeline_progress`
+    and should be forwarded to the pipeline helpers' ``on_progress`` /
+    ``on_progress_console`` parameters. ``console`` is the renderer's stderr
+    ``rich.Console`` so the subprocess drain routes non-progress lines through
+    ``console.print`` while the Live region is up (single-writer invariant).
+    """
+
+    def __init__(self, *, renderer: "object | None") -> None:
+        self.renderer = renderer
+        self.on_progress: "Callable | None" = None
+        self.console: "object | None" = None
+
+
 def _jsonable(value: Any) -> Any:
     if hasattr(value, "model_dump"):
         return value.model_dump()
@@ -266,7 +309,7 @@ def _cmd_init(args: argparse.Namespace) -> int:
         return 2
     cfg.index_dir.mkdir(parents=True, exist_ok=True)
 
-    def work() -> int:
+    def work(progress: "PipelineProgress | None") -> int:
         env = cfg.subprocess_env()
         verbose = bool(args.verbose)
         coco = run_cocoindex_update(
@@ -275,6 +318,8 @@ def _cmd_init(args: argparse.Namespace) -> int:
             quiet=bool(args.quiet),
             verbose=verbose,
             lance_project_root=None if args.quiet else cfg.source_root,
+            on_progress=progress.on_progress if progress is not None else None,
+            on_progress_console=progress.console if progress is not None else None,
         )
         if coco.returncode != 0:
             _emit(
@@ -295,6 +340,8 @@ def _cmd_init(args: argparse.Namespace) -> int:
             verbose=verbose,
             quiet=bool(args.quiet),
             env=env,
+            on_progress=progress.on_progress if progress is not None else None,
+            on_progress_console=progress.console if progress is not None else None,
         )
         if g.returncode != 0:
             _emit(
@@ -310,7 +357,9 @@ def _cmd_init(args: argparse.Namespace) -> int:
         _emit({"success": True, "message": "init completed"})
         return 0
 
-    return _run_with_pipeline_progress("init", cfg, quiet=bool(args.quiet), work=work)
+    return _run_with_pipeline_progress(
+        "init", cfg, quiet=bool(args.quiet), verbose=bool(args.verbose), work=work
+    )
 
 
 def _cmd_increment(args: argparse.Namespace) -> int:
@@ -323,7 +372,7 @@ def _cmd_increment(args: argparse.Namespace) -> int:
     if vectors_only:
         _emit_increment_ladybug_warning()
 
-    def work() -> int:
+    def work(progress: "PipelineProgress | None") -> int:
         env = cfg.subprocess_env()
         coco = run_cocoindex_update(
             env,
@@ -331,6 +380,8 @@ def _cmd_increment(args: argparse.Namespace) -> int:
             quiet=bool(args.quiet),
             verbose=bool(args.verbose),
             lance_project_root=None if args.quiet else cfg.source_root,
+            on_progress=progress.on_progress if progress is not None else None,
+            on_progress_console=progress.console if progress is not None else None,
         )
         if coco.returncode != 0:
             _emit(
@@ -356,6 +407,8 @@ def _cmd_increment(args: argparse.Namespace) -> int:
             verbose=bool(args.verbose),
             quiet=bool(args.quiet),
             env=env,
+            on_progress=progress.on_progress if progress is not None else None,
+            on_progress_console=progress.console if progress is not None else None,
         )
 
         # Check if incremental fell back to full rebuild
@@ -389,7 +442,9 @@ def _cmd_increment(args: argparse.Namespace) -> int:
         _emit({"success": True, "message": "increment completed (Lance + graph updated)"})
         return 0
 
-    return _run_with_pipeline_progress("increment", cfg, quiet=bool(args.quiet), work=work)
+    return _run_with_pipeline_progress(
+        "increment", cfg, quiet=bool(args.quiet), verbose=bool(args.verbose), work=work
+    )
 
 
 def _cmd_reprocess(args: argparse.Namespace) -> int:
@@ -397,14 +452,18 @@ def _cmd_reprocess(args: argparse.Namespace) -> int:
     _startup_hints(cfg)
     cfg.apply_to_os_environ()
 
-    def work() -> int:
+    def work(progress: "PipelineProgress | None") -> int:
         env = cfg.subprocess_env()
         verbose = bool(args.verbose)
         vectors_only = bool(getattr(args, "vectors_only", False))
         graph_only = bool(getattr(args, "graph_only", False))
 
         if vectors_only:
-            coco = run_cocoindex_update(env, full_reprocess=True, quiet=bool(args.quiet), verbose=verbose)
+            coco = run_cocoindex_update(
+                env, full_reprocess=True, quiet=bool(args.quiet), verbose=verbose,
+                on_progress=progress.on_progress if progress is not None else None,
+                on_progress_console=progress.console if progress is not None else None,
+            )
             if _is_cocoindex_preflight_blocker(coco):
                 payload: dict[str, Any] = {
                     "success": False,
@@ -443,6 +502,8 @@ def _cmd_reprocess(args: argparse.Namespace) -> int:
                 verbose=verbose,
                 quiet=bool(args.quiet),
                 env=env,
+                on_progress=progress.on_progress if progress is not None else None,
+                on_progress_console=progress.console if progress is not None else None,
             )
             if _is_graph_preflight_blocker(g):
                 payload = {
@@ -477,12 +538,21 @@ def _cmd_reprocess(args: argparse.Namespace) -> int:
 
         import server  # lazy: pulls sentence_transformers/torch/lancedb/kuzu
 
-        result = asyncio.run(server.run_refresh_pipeline(quiet=bool(args.quiet), verbose=verbose))
+        result = asyncio.run(
+            server.run_refresh_pipeline(
+                quiet=bool(args.quiet),
+                verbose=verbose,
+                on_progress=progress.on_progress if progress is not None else None,
+                on_progress_console=progress.console if progress is not None else None,
+            )
+        )
         payload = result.model_dump()
         _emit_reprocess_outcome(payload)
         return _reprocess_exit_code(payload)
 
-    return _run_with_pipeline_progress("reprocess", cfg, quiet=bool(args.quiet), work=work)
+    return _run_with_pipeline_progress(
+        "reprocess", cfg, quiet=bool(args.quiet), verbose=bool(args.verbose), work=work
+    )
 
 
 def _cmd_install(args: argparse.Namespace) -> int:
@@ -495,6 +565,7 @@ def _cmd_install(args: argparse.Namespace) -> int:
         model=args.model,
         source_root=None,  # None means cwd; installer confirms interactively
         quiet=bool(args.quiet),
+        verbose=bool(args.verbose),
     )
 
 
@@ -504,6 +575,8 @@ def _cmd_update(args: argparse.Namespace) -> int:
     return run_update(
         force=bool(args.force),
         dry_run=bool(args.dry_run),
+        quiet=bool(args.quiet),
+        verbose=bool(args.verbose),
     )
 
 
@@ -537,7 +610,7 @@ def _cmd_erase(args: argparse.Namespace) -> int:
             print("Aborted.", file=sys.stderr)
             return 2
 
-    def work() -> int:
+    def work(progress: "PipelineProgress | None") -> int:
         env = cfg.subprocess_env()
         drop = run_cocoindex_drop(env, quiet=bool(args.quiet))
         if drop.returncode == 127:
@@ -570,7 +643,7 @@ def _cmd_erase(args: argparse.Namespace) -> int:
         _emit({"success": True, "message": "erase completed"})
         return 0
 
-    return _run_with_pipeline_progress("erase", cfg, quiet=bool(args.quiet), work=work)
+    return _run_with_pipeline_progress("erase", cfg, quiet=bool(args.quiet), verbose=bool(getattr(args, "verbose", False)), work=work)
 
 
 def _cmd_meta(args: argparse.Namespace) -> int:

java_codebase_rag/cli_format.py

@@ -1,10 +1,7 @@
 """TTY-aware ANSI formatting for CLI stderr progress."""
 from __future__ import annotations
 
-import itertools
 import sys
-import threading
-import time
 
 _RESET = "\033[0m"
 _BOLD = "\033[1m"
@@ -16,8 +13,6 @@ _CYAN = "\033[36m"
 CHECK = "✓"
 CROSS = "✗"
 
-_SPINNER_FRAMES = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
-
 _NOISE_CONTAINS: tuple[bytes, ...] = (
     b"lance::",
     b"FutureWarning",
@@ -25,6 +20,14 @@ _NOISE_CONTAINS: tuple[bytes, ...] = (
     b'"event": "brownfield-',
     b"unknown producer source strategy",
     b"unknown client source strategy",
+    # Builder verbose heartbeats / pass banners: in default mode the renderer's
+    # bar subsumes these, so they must NOT also appear as raw lines above the
+    # Live region. --verbose raw-relay bypasses this filter and still shows them.
+    b"[graph] pass ",
+    b"[graph] scoped write ",
+    b"[graph] writing ",
+    b"[graph] done ",
+    b"[increment] ",
 )
 
 
@@ -80,33 +83,3 @@ def styled_check() -> str:
 
 def styled_cross() -> str:
     return red(CROSS) if stderr_is_tty() else CROSS
-
-
-class Spinner:
-    """Braille spinner that overwrites the current stderr line until stopped."""
-
-    def __init__(self, label: str) -> None:
-        self._label = label
-        self._stop = threading.Event()
-        self._thread: threading.Thread | None = None
-
-    def start(self) -> None:
-        self._thread = threading.Thread(target=self._run, name="spinner", daemon=True)
-        self._thread.start()
-
-    def stop(self) -> None:
-        self._stop.set()
-        if self._thread is not None:
-            self._thread.join(timeout=2.0)
-        sys.stderr.buffer.write(b"\r\x1b[2K")
-        sys.stderr.buffer.flush()
-
-    def _run(self) -> None:
-        frames = itertools.cycle(_SPINNER_FRAMES)
-        t0 = time.monotonic()
-        while not self._stop.wait(0.3):
-            elapsed = time.monotonic() - t0
-            frame = next(frames)
-            line = f"\r{frame} {self._label} · {elapsed:.0f}s"
-            sys.stderr.buffer.write(line.encode())
-            sys.stderr.buffer.flush()

java_codebase_rag/cli_progress.py

@@ -3,26 +3,10 @@ from __future__ import annotations
 
 import asyncio
 import sys
+from typing import Callable
 
-from java_codebase_rag.cli_format import bold_cyan, is_noise_line, styled_check, styled_cross
-
-
-def emit_vectors_start() -> None:
-    print(
-        bold_cyan("[vectors]") + " running · cocoindex update",
-        file=sys.stderr,
-        flush=True,
-    )
-
-
-def emit_vectors_finish(*, elapsed_s: float, exit_code: int) -> None:
-    marker = styled_check() if exit_code == 0 else styled_cross()
-    print(
-        f"{marker} {bold_cyan('[vectors]')} finished · {elapsed_s:.2f}s"
-        + (f" (exit={exit_code})" if exit_code != 0 else ""),
-        file=sys.stderr,
-        flush=True,
-    )
+from java_codebase_rag.cli_format import is_noise_line
+from java_codebase_rag.progress import ProgressEvent, make_relay
 
 
 class _AsyncLineFilter:
@@ -61,8 +45,15 @@ async def accumulate_and_relay_subprocess_streams(
     *,
     relay: bool,
     verbose: bool = True,
+    on_progress: Callable[[ProgressEvent], None] | None = None,
+    on_progress_console: object | None = None,
 ) -> tuple[bytes, bytes]:
-    """Read stdout and stderr until EOF; optionally copy non-noise stderr chunks to stderr."""
+    """Read stdout and stderr until EOF; optionally copy non-noise stderr chunks to stderr.
+
+    When ``on_progress`` is set, stderr is drained through a :class:`ProgressRelay`
+    so ``JCIRAG_PROGRESS`` lines are parsed and routed to ``on_progress`` (and
+    suppressed from the relay), matching the sync ``pipeline._popen_capturing_stderr``.
+    """
     stdout = proc.stdout
     stderr = proc.stderr
     if stdout is None or stderr is None:
@@ -70,7 +61,12 @@ async def accumulate_and_relay_subprocess_streams(
 
     out_buf = bytearray()
     err_buf = bytearray()
-    filt = _AsyncLineFilter() if (relay and not verbose) else None
+    if on_progress is not None:
+        filt = make_relay(on_progress, console=on_progress_console, verbose=verbose)
+    elif relay and not verbose:
+        filt = _AsyncLineFilter()
+    else:
+        filt = None
 
     async def drain_stdout(reader: asyncio.StreamReader, target: bytearray) -> None:
         while True: