java-codebase-rag 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

ast_java.py

@@ -14,8 +14,8 @@ from __future__ import annotations
 
 import posixpath
 import sys
+import threading
 from dataclasses import dataclass, field
-from functools import lru_cache
 from typing import Iterable
 
 import tree_sitter_java as _ts_java
@@ -215,10 +215,23 @@ _ANON_SUPER_AS_INTERFACE: frozenset[str] = frozenset({
 })
 
 
-@lru_cache(maxsize=1)
+# tree-sitter's ``Parser`` mutates internal state during ``parse()`` and is NOT
+# thread-safe, so each OS thread gets its own instance. ``parse_java`` is called
+# concurrently from worker threads when indexing runs with cocoindex's inflight
+# parallelism — both directly (java_index_flow_lancedb.py: process_java_file
+# offloads parse+enrich to asyncio.to_thread) and transitively
+# (graph_enrich.collect_annotation_meta_chain -> _collect_annotation_decl_index
+# -> parse_java, reached from enrich_chunk). The ``Language`` is immutable and
+# shared; per-thread ``Parser`` construction is lazy and cheap (once per thread),
+# which also preserves parse parallelism instead of serializing it.
+_parser_tls = threading.local()
+
+
 def _parser() -> Parser:
-    lang = Language(_ts_java.language())
-    return Parser(lang)
+    p = getattr(_parser_tls, "parser", None)
+    if p is None:
+        _parser_tls.parser = p = Parser(Language(_ts_java.language()))
+    return p
 
 
 # ---------- dataclasses ----------
@@ -1565,62 +1578,20 @@ def _parse_codebase_http_route_inner_annotation(
     return out
 
 
-def _codebase_route_inner_annotation_nodes(container_ann: Node, src: bytes) -> list[Node]:
-    found: list[Node] = []
-
-    def visit(n: Node) -> None:
-        if n.type == "annotation":
-            name_node = n.child_by_field_name("name")
-            n_simple = _txt(name_node, src).rsplit(".", 1)[-1] if name_node is not None else ""
-            if n_simple == "CodebaseHttpRoute":
-                found.append(n)
-        for c in n.children:
-            visit(c)
+def _inner_annotation_nodes(container_ann: Node, src: bytes, target_simple: str) -> list[Node]:
+    """Collect nested ``@<target_simple>`` annotations anywhere under ``container_ann``.
 
-    visit(container_ann)
-    return found
-
-
-def _codebase_async_route_inner_annotation_nodes(container_ann: Node, src: bytes) -> list[Node]:
-    found: list[Node] = []
-
-    def visit(n: Node) -> None:
-        if n.type == "annotation":
-            name_node = n.child_by_field_name("name")
-            n_simple = _txt(name_node, src).rsplit(".", 1)[-1] if name_node is not None else ""
-            if n_simple == "CodebaseAsyncRoute":
-                found.append(n)
-        for c in n.children:
-            visit(c)
-
-    visit(container_ann)
-    return found
-
-
-def _codebase_http_client_inner_annotation_nodes(container_ann: Node, src: bytes) -> list[Node]:
-    found: list[Node] = []
-
-    def visit(n: Node) -> None:
-        if n.type == "annotation":
-            name_node = n.child_by_field_name("name")
-            n_simple = _txt(name_node, src).rsplit(".", 1)[-1] if name_node is not None else ""
-            if n_simple == "CodebaseHttpClient":
-                found.append(n)
-        for c in n.children:
-            visit(c)
-
-    visit(container_ann)
-    return found
-
-
-def _codebase_producer_inner_annotation_nodes(container_ann: Node, src: bytes) -> list[Node]:
+    Shared by the four brownfield container walkers — ``CodebaseHttpRoute``,
+    ``CodebaseAsyncRoute``, ``CodebaseHttpClient``, ``CodebaseProducer`` — which
+    differ only by the target annotation simple name.
+    """
     found: list[Node] = []
 
     def visit(n: Node) -> None:
         if n.type == "annotation":
             name_node = n.child_by_field_name("name")
             n_simple = _txt(name_node, src).rsplit(".", 1)[-1] if name_node is not None else ""
-            if n_simple == "CodebaseProducer":
+            if n_simple == target_simple:
                 found.append(n)
         for c in n.children:
             visit(c)
@@ -1842,7 +1813,7 @@ def _outgoing_calls_from_codebase_http_client_producer_annotations(
                 ),
             )
         elif simple == "CodebaseHttpClients":
-            for inner in _codebase_http_client_inner_annotation_nodes(ann, src):
+            for inner in _inner_annotation_nodes(ann, src, "CodebaseHttpClient"):
                 out.append(
                     _parse_codebase_http_client_annotation(
                         inner,
@@ -1869,7 +1840,7 @@ def _outgoing_calls_from_codebase_http_client_producer_annotations(
                 ),
             )
         elif simple == "CodebaseProducers":
-            for inner in _codebase_producer_inner_annotation_nodes(ann, src):
+            for inner in _inner_annotation_nodes(ann, src, "CodebaseProducer"):
                 out.append(
                     _parse_codebase_producer_annotation(
                         inner,
@@ -2343,7 +2314,7 @@ def _collect_routes(
                 ),
             )
         elif simple == "CodebaseHttpRoutes":
-            for inner in _codebase_route_inner_annotation_nodes(node, src):
+            for inner in _inner_annotation_nodes(node, src, "CodebaseHttpRoute"):
                 routes.extend(
                     _parse_codebase_http_route_inner_annotation(
                         inner,
@@ -2359,7 +2330,7 @@ def _collect_routes(
         elif simple in ("CodebaseAsyncRoute", "CodebaseAsyncRoutes"):
             nodes = [node]
             if simple == "CodebaseAsyncRoutes":
-                nodes = list(_codebase_async_route_inner_annotation_nodes(node, src))
+                nodes = list(_inner_annotation_nodes(node, src, "CodebaseAsyncRoute"))
             for ann in nodes:
                 pairs, _ = _annotation_kv_nodes(ann, src)
                 topic_node = pairs.get("topic")

build_ast_graph.py

@@ -25,6 +25,7 @@ The LadybugDB DB is dropped and rebuilt on every run (Phase 1 is a full rebuild)
 from __future__ import annotations
 
 import argparse
+import contextlib
 import hashlib
 import json
 import logging
@@ -84,6 +85,53 @@ def _verbose_stderr_line(content: str) -> None:
         print(content, file=sys.stderr, flush=True)
 
 
+def _emit_graph_progress(parts: dict[str, object], *, verbose: bool) -> None:
+    """Emit one ``JCIRAG_PROGRESS kind=graph …`` line to stderr (gated by verbose).
+
+    The parent process (``pipeline.run_build_ast_graph`` /
+    ``run_incremental_graph``) passes ``--verbose`` in default AND verbose modes
+    (only suppressed for ``--quiet``), so this structured progress surfaces in
+    default mode (where the parent renders it) and verbose mode (raw relay). In
+    ``--quiet`` the builder is never invoked with ``--verbose`` so nothing is
+    emitted. Field order is fixed so the parser and tests can pin substrings.
+    """
+    if not verbose:
+        return
+    fields = ["kind=graph"]
+    for key in ("pass", "done", "total", "status", "elapsed_s"):
+        if key in parts:
+            fields.append(f"{key}={parts[key]}")
+    line = "JCIRAG_PROGRESS " + " ".join(fields)
+    _verbose_stderr_line(line)
+
+
+# Pass-1 per-file tick cadence: bound stderr volume on huge trees without making
+# the bar feel stale. A final tick on pass completion carries status=done.
+_PASS1_TICK_EVERY = 25
+
+
+@contextlib.contextmanager
+def _graph_pass_progress(pass_label: str, *, verbose: bool):
+    """Emit ``pass=N/6 status=running`` on entry and ``status=done elapsed_s=…``
+    on exit for passes 2–6 (each advances the rendered bar by 1/6).
+
+    Usage: ``with _graph_pass_progress("2/6", verbose=verbose): …``
+    """
+    if not verbose:
+        yield
+        return
+    _emit_graph_progress({"pass": pass_label, "status": "running"}, verbose=verbose)
+    t0 = time.time()
+    try:
+        yield
+    finally:
+        elapsed = time.time() - t0
+        _emit_graph_progress(
+            {"pass": pass_label, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
+            verbose=verbose,
+        )
+
+
 class _VerbosePassHeartbeats:
     """Emit ``[tag] running … Ns elapsed`` every 5s on stderr while in scope (verbose only)."""
 
@@ -837,7 +885,14 @@ def _register_type(
     return entry
 
 
-def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files: set[str] | None = None) -> dict[str, JavaFileAst]:
+def pass1_parse(
+    root: Path,
+    tables: GraphTables,
+    *,
+    verbose: bool,
+    scope_files: set[str] | None = None,
+    removed_files: set[str] | None = None,
+) -> dict[str, JavaFileAst]:
     """Walk files, parse them, populate node indexes. Returns path -> AST.
 
     Args:
@@ -845,6 +900,11 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
         tables: GraphTables to populate.
         verbose: Whether to emit progress output.
         scope_files: Optional set of relative POSIX paths to parse. If None, parse all files.
+        removed_files: Optional set of relative POSIX paths that no longer exist
+            on disk (incremental deletions). These are members of ``scope_files``
+            (they were deleted, so they participate in scoped deletion) but are
+            never visited by the parse walk, so they must be excluded from the
+            pass-1 total to keep ``done`` from undercounting then two-way-clamping.
     """
     asts: dict[str, JavaFileAst] = {}
     ignore = LayeredIgnore(root)
@@ -852,6 +912,23 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
     n_files = 0
     if verbose:
         _verbose_stderr_line(_PASS1_START)
+    # Count-first: one filtered walk (no parsing) to set the EXACT total before
+    # the parse loop ticks. Single-layer ignore → the count is exact, so the
+    # rendered bar is determinate. For a scoped (incremental) parse the total is
+    # the number of files that will actually be visited: scope minus any removed
+    # files (which are members of scope for deletion but gone from disk, so the
+    # parse walk never ticks them); for a full rebuild it is the non-ignored
+    # .java count.
+    if verbose:
+        if scope_files is not None:
+            removed = removed_files if removed_files is not None else set()
+            pass1_total = len(scope_files - removed)
+        else:
+            pass1_total = sum(1 for _ in iter_java_source_files(root, ignore=ignore))
+        _emit_graph_progress(
+            {"pass": "1/6", "done": 0, "total": pass1_total, "status": "running"},
+            verbose=verbose,
+        )
     slow_sec = 0.0
     raw_slow = os.environ.get("JAVA_CODEBASE_RAG_TEST_GRAPH_SLOW_SEC", "").strip()
     if raw_slow:
@@ -871,6 +948,11 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
             if scope_files is not None and rel not in scope_files:
                 continue
             n_files += 1
+            if verbose and (n_files % _PASS1_TICK_EVERY == 0):
+                _emit_graph_progress(
+                    {"pass": "1/6", "done": n_files, "status": "running"},
+                    verbose=verbose,
+                )
             try:
                 content = p.read_bytes()
             except OSError:
@@ -906,6 +988,10 @@ def pass1_parse(root: Path, tables: GraphTables, *, verbose: bool, scope_files:
 
     if verbose:
         elapsed = time.time() - t0
+        _emit_graph_progress(
+            {"pass": "1/6", "done": n_files, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
+            verbose=verbose,
+        )
         _verbose_stderr_line(
             f"[graph] pass 1 · parsed {n_files} files in {elapsed:.2f}s: "
             f"{len(tables.types)} types, {len(tables.members)} members, "
@@ -1145,7 +1231,7 @@ def pass2_edges(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: b
     seen_inj: set[tuple[str, str, str, str]] = set()
     if verbose:
         _verbose_stderr_line(_PASS2_START)
-    with _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
+    with _graph_pass_progress("2/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
         for fqn, entry in tables.types.items():
             ast = asts.get(entry.file_path)
             if ast is None:
@@ -1818,7 +1904,7 @@ def pass3_calls(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: b
         _verbose_stderr_line(_PASS3_START)
     _build_member_indexes(tables)
     stats = CallResolutionStats()
-    with _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
+    with _graph_pass_progress("3/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
         for rel_path, file_ast in asts.items():
             try:
                 _process_file_calls(file_ast, rel_path, tables, stats)
@@ -1924,8 +2010,21 @@ def _producer_id(
     return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
 
 
+# The four brownfield source layers — single source of truth. Consumed by the
+# client/producer source-layer classifiers, the *_from_brownfield_pct stats
+# (via brownfield_strategies), and the brownfield_only authoritativeness gate in
+# _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
+# declaration strategies, not layers — they extend brownfield_strategies only.
+_BROWNFIELD_LAYERS = frozenset({
+    "layer_a_meta",
+    "layer_b_ann",
+    "layer_b_fqn",
+    "layer_c_source",
+})
+
+
 def _client_source_layer(strategy: str) -> str:
-    if strategy in {"layer_a_meta", "layer_b_ann", "layer_b_fqn", "layer_c_source"}:
+    if strategy in _BROWNFIELD_LAYERS:
         return strategy
     # Some caller extraction paths emit client kind as strategy; treat those
     # as builtin-source declarations instead of warning on every row.
@@ -1937,7 +2036,7 @@ def _client_source_layer(strategy: str) -> str:
 
 
 def _producer_source_layer(strategy: str) -> str:
-    if strategy in {"layer_a_meta", "layer_b_ann", "layer_b_fqn", "layer_c_source"}:
+    if strategy in _BROWNFIELD_LAYERS:
         return strategy
     if strategy in VALID_PRODUCER_KINDS:
         return "builtin"
@@ -1972,7 +2071,7 @@ def pass4_routes(
     meta_chain = collect_annotation_meta_chain(prs)
     if verbose:
         _verbose_stderr_line(_PASS4_START)
-    with _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
+    with _graph_pass_progress("4/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
 
         for ast in asts.values():
             stats.routes_skipped_unresolved += ast.routes_skipped_unresolved
@@ -2149,7 +2248,7 @@ def pass5_imperative_edges(
 
     if verbose:
         _verbose_stderr_line(_PASS5_START)
-    with _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
+    with _graph_pass_progress("5/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
         for member in sorted(tables.members, key=lambda x: x.node_id):
             if member.decl.is_constructor:
                 continue
@@ -2372,15 +2471,14 @@ def pass5_imperative_edges(
         tables.producer_stats.producers_by_kind = defaultdict(int)
         for row in tables.producer_rows:
             tables.producer_stats.producers_by_kind[row.producer_kind] += 1
-        brownfield_strategies = frozenset(
-            (
-                "layer_b_ann",
-                "layer_a_meta",
-                "layer_c_source",
-                "layer_b_fqn",
-                "codebase_client",
-                "codebase_producer",
-            ),
+        # brownfield_strategies = the four brownfield layers plus the two
+        # caller-side declaration strategies (@CodebaseHttpClient /
+        # @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
+        # the *_from_brownfield_pct stats count annotation-declared callers as
+        # brownfield-sourced even though they are not "layers" and so do not
+        # gate brownfield_only authoritativeness in _is_brownfield_sourced.
+        brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
+            {"codebase_client", "codebase_producer"},
         )
         if tables.call_edge_stats.http_calls_total:
             n_http = sum(
@@ -2482,14 +2580,6 @@ def _match_call_edge(
     return "cross_service", candidates
 
 
-_BROWNFIELD_LAYERS = frozenset({
-    "layer_c_source",
-    "layer_b_ann",
-    "layer_b_fqn",
-    "layer_a_meta",
-})
-
-
 def _is_brownfield_sourced(
     call_strategy: str,
     candidates: list[RouteRow],
@@ -2551,7 +2641,7 @@ def pass6_match_edges(
 
     if verbose:
         _verbose_stderr_line(_PASS6_START)
-    with _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
+    with _graph_pass_progress("6/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
         for row in tables.http_call_rows:
             if row.match != "unresolved":
                 continue
@@ -3586,7 +3676,9 @@ def incremental_rebuild(
             _verbose_stderr_line("[increment] rebuilding scoped files (passes 1-4)")
 
         tables = GraphTables()
-        asts = pass1_parse(source_root, tables, verbose=verbose, scope_files=scope_files)
+        asts = pass1_parse(
+            source_root, tables, verbose=verbose, scope_files=scope_files, removed_files=removed
+        )
 
         # Load existing types and members for cross-file resolution (only from unchanged files)
         _load_existing_types(conn, tables, exclude_files=scope_files)

graph_enrich.py

@@ -23,7 +23,7 @@ import sys
 from dataclasses import dataclass, field, replace
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 from ast_java import (
     AnnotationRef,
     JavaFileAst,
@@ -820,7 +820,15 @@ def _route_path_atom(raw_value: str, value_kind: str | None) -> tuple[str, str,
     return "", "constant_ref", 0.7, False
 
 
-def _route_hint_lookup(ann: AnnotationRef, hints: dict[str, RouteHint]) -> RouteHint | None:
+_HINT = TypeVar("_HINT")
+
+
+def _hint_lookup(ann: AnnotationRef, hints: dict[str, _HINT]) -> _HINT | None:
+    """Resolve a brownfield hint by qualified name, then simple name, then suffix.
+
+    Shared by route / http-client / async-producer hint resolution; the three
+    former copies differed only in the hint value type.
+    """
     q = ann.qualified.strip()
     if q in hints:
         return hints[q]
@@ -1118,7 +1126,7 @@ def resolve_routes_for_method(
 
     # ----- Step 2: Layer B — annotation route hints -----
     for _is_m, ann in combined_anns:
-        hint = _route_hint_lookup(ann, overrides.annotation_to_route_hint)
+        hint = _hint_lookup(ann, overrides.annotation_to_route_hint)
         if hint is None:
             continue
         working.append(
@@ -1172,36 +1180,6 @@ def resolve_routes_for_method(
     return working
 
 
-def _client_hint_lookup(
-    ann: AnnotationRef,
-    hints: dict[str, HttpClientHint],
-) -> HttpClientHint | None:
-    q = ann.qualified.strip()
-    if q in hints:
-        return hints[q]
-    if ann.name in hints:
-        return hints[ann.name]
-    for k, h in sorted(hints.items(), key=lambda kv: kv[0]):
-        if k.endswith("." + ann.name):
-            return h
-    return None
-
-
-def _async_hint_lookup(
-    ann: AnnotationRef,
-    hints: dict[str, AsyncProducerHint],
-) -> AsyncProducerHint | None:
-    q = ann.qualified.strip()
-    if q in hints:
-        return hints[q]
-    if ann.name in hints:
-        return hints[ann.name]
-    for k, h in sorted(hints.items(), key=lambda kv: kv[0]):
-        if k.endswith("." + ann.name):
-            return h
-    return None
-
-
 def _call_from_http_hint(
     *,
     hint: HttpClientHint,
@@ -1296,7 +1274,7 @@ def resolve_http_client_for_method(
     anchor = builtin_http[0] if builtin_http else (layer_c_src[0] if layer_c_src else None)
 
     for _is_m, ann in combined_anns:
-        hint = _client_hint_lookup(ann, overrides.annotation_to_http_client_hint)
+        hint = _hint_lookup(ann, overrides.annotation_to_http_client_hint)
         if hint is None:
             continue
         brownfield_calls.append(
@@ -1388,7 +1366,7 @@ def resolve_async_producer_for_method(
     anchor = builtin_async[0] if builtin_async else (layer_c_src[0] if layer_c_src else None)
 
     for _is_m, ann in combined_anns:
-        hint = _async_hint_lookup(ann, overrides.annotation_to_async_producer_hint)
+        hint = _hint_lookup(ann, overrides.annotation_to_async_producer_hint)
         if hint is None:
             continue
         brownfield_calls.append(