java-codebase-rag 0.9.4__py3-none-any.whl → 0.9.6__py3-none-any.whl

java_index_flow_lancedb.py → java_codebase_rag/index/java_index_flow_lancedb.py

@@ -41,7 +41,7 @@ from cocoindex.resources.file import PatternFilePathMatcher
 
 from java_codebase_rag.config import resolved_sbert_model_for_process_env
 from java_codebase_rag.lance_optimize import LANCE_TABLE_NAMES
-from java_index_v1_common import (
+from java_codebase_rag.index.java_index_v1_common import (
     JAVA_CHUNK,
     SBERT_MODEL,
     SQL_CHUNK,
@@ -49,9 +49,15 @@ from java_index_v1_common import (
     chunk_key_range,
     position_to_json,
 )
-from path_filtering import LayeredIgnore
-from ast_java import ONTOLOGY_VERSION, parse_java
-from graph_enrich import collect_annotation_meta_chain, enrich_chunk, load_brownfield_overrides
+from java_codebase_rag.graph.path_filtering import LayeredIgnore
+from java_codebase_rag.ast.ast_java import ONTOLOGY_VERSION, parse_java
+from java_codebase_rag.graph.graph_enrich import (
+    classify_java_file,
+    collect_annotation_meta_chain,
+    enrich_chunk,
+    load_brownfield_overrides,
+    load_generated_detection,
+)
 
 # Older cocoindex (e.g. 1.0.0a43) uses ``tracked=False``; newer releases renamed
 # the flag to ``detect_change`` (default False) and reject ``tracked``.
@@ -280,6 +286,9 @@ class JavaLanceChunk:
     annotations_on_type: Annotated[list[str], LanceType(pa.list_(pa.string()))]
     symbols: Annotated[list[str], LanceType(pa.list_(pa.string()))]
     ontology_version: int
+    # Generated source detection: populated per-file, not per-chunk
+    generated: bool
+    generated_by: str | None
 
 
 @dataclass
@@ -364,12 +373,13 @@ def _parse_and_enrich_java(
     chunks: list[Any],
     rel: str,
     project_root: Path,
-) -> list[Any]:
+) -> tuple[list[Any], Any]:
     """Parse one Java file and enrich every chunk, off the event loop.
 
-    Returns a list of :class:`graph_enrich.ChunkEnrichment` aligned 1:1 with
-    ``chunks``. Intended to run via ``asyncio.to_thread`` from
-    ``process_java_file`` (vectors perf lever #2): while the worker thread
+    Returns a tuple of (enrichments, ast) where enrichments is a list of
+    :class:`graph_enrich.ChunkEnrichment` aligned 1:1 with ``chunks``, and ast
+    is the parsed :class:`JavaFileAst`. Intended to run via ``asyncio.to_thread``
+    from ``process_java_file`` (vectors perf lever #2): while the worker thread
     parses + enriches, the event loop is free to drive other files and keep the
     embedder's batching queue fed.
 
@@ -381,7 +391,7 @@ def _parse_and_enrich_java(
     ``lru_cache`` reads are thread-safe under the GIL.
     """
     ast = parse_java(content_bytes)
-    return [
+    enrichments = [
         enrich_chunk(
             ast,
             chunk_start_byte=ch.start.byte_offset,
@@ -391,6 +401,7 @@ def _parse_and_enrich_java(
         )
         for ch in chunks
     ]
+    return enrichments, ast
 
 
 @coco.fn(memo=True)
@@ -430,9 +441,16 @@ async def process_java_file(
     # (vectors perf lever #2) parse + enrich off the event loop so the loop can
     # keep the embedder's batching queue fed while this file is being parsed.
     # parse_java is thread-safe (per-thread tree-sitter Parser in ast_java).
-    enrichments = await asyncio.to_thread(
+    enrichments, ast = await asyncio.to_thread(
         _parse_and_enrich_java, content_bytes, chunks, rel, project_root
     )
+
+    # Compute generated source detection once per file (uses the AST and content_bytes)
+    generated_config = load_generated_detection(project_root)
+    generated, generated_by = classify_java_file(
+        content_bytes, ast, config=generated_config, project_root=project_root
+    )
+
     # (vectors perf lever #1) embed all chunks concurrently so the batched
     # embedder groups them into one ``model.encode(...)`` (max_batch_size=64)
     # instead of N serial batch-of-1 calls. Dominant win for ``increment``
@@ -462,6 +480,8 @@ async def process_java_file(
                 annotations_on_type=enrich.annotations_on_type,
                 symbols=enrich.symbols,
                 ontology_version=ONTOLOGY_VERSION,
+                generated=generated,
+                generated_by=generated_by,
             )
         )
 

java_codebase_rag/jrag.py

@@ -134,7 +134,7 @@ def _apply_auto_scope(args: argparse.Namespace, cfg, graph) -> None:
     source_root = cfg.source_root if cfg.source_root else None
     if not source_root:
         return
-    from graph_enrich import detect_microservice_from_path
+    from java_codebase_rag.graph.graph_enrich import detect_microservice_from_path
 
     candidate = detect_microservice_from_path(Path.cwd(), Path(source_root))
     if not candidate:
@@ -1163,6 +1163,20 @@ def build_parser() -> argparse.ArgumentParser:
     )
     search.set_defaults(handler=_cmd_search, auto_scope=True)
 
+    # ---- vocab-index subparser (PR-ABS-1) ----
+    vocab_index = subparsers.add_parser(
+        "vocab-index",
+        help="Rebuild the vocabulary index (absence diagnosis).",
+        parents=[_core_parser()],
+        description=(
+            "Rebuild the vocabulary index sidecar from the current Ladybug graph. "
+            "The index is a search-optimized projection of Symbol nodes used for "
+            "did-you-mean suggestions and external membership checks in absence "
+            "diagnosis. Printed on success: symbol count and sidecar path."
+        ),
+    )
+    vocab_index.set_defaults(handler=_cmd_vocab_index, detail="full")
+
     return parser
 
 
@@ -1203,7 +1217,7 @@ def _load_graph(cfg):  # type: ignore[no-untyped-def]
     * ontology-mismatch (``RuntimeError`` from ``LadybugGraph.get``) ->
       ``_IndexStale`` (caught in ``main`` -> envelope with a rebuild hint).
     """
-    from ladybug_queries import LadybugGraph
+    from java_codebase_rag.graph.ladybug_queries import LadybugGraph
 
     ladybug_path = str(cfg.ladybug_path)
     if not LadybugGraph.exists(ladybug_path):
@@ -1217,6 +1231,40 @@ def _load_graph(cfg):  # type: ignore[no-untyped-def]
         raise _IndexStale(str(exc)) from exc
 
 
+def _cmd_vocab_index(args: argparse.Namespace) -> int:
+    """Rebuild the vocabulary index sidecar from the Ladybug graph."""
+    from java_codebase_rag.ast.ast_java import ONTOLOGY_VERSION
+    from java_codebase_rag.absence.absence_vocab import VocabularyIndex, VOCAB_INDEX_FILENAME
+
+    cfg = _resolve_cfg(args)
+    try:
+        graph = _load_graph(cfg)
+    except (_IndexNotFound, _IndexStale) as exc:
+        print(f"[error] {exc}", file=sys.stderr)
+        return 2
+
+    # Build vocabulary index
+    try:
+        index = VocabularyIndex.build(graph, q=cfg.absence_ngram_q)
+    except Exception as e:
+        print(f"[error] Vocabulary index build failed: {e}", file=sys.stderr)
+        return 1
+
+    # Save to sidecar
+    sidecar_path = cfg.ladybug_path.parent / VOCAB_INDEX_FILENAME
+    try:
+        index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION)
+    except Exception as e:
+        print(f"[error] Failed to save vocabulary index: {e}", file=sys.stderr)
+        return 1
+
+    # Print success message (simple format for admin command)
+    print(f"Vocabulary index rebuilt successfully:")
+    print(f"  Symbol count: {index.symbol_count}")
+    print(f"  Sidecar path: {sidecar_path}")
+    return 0
+
+
 def _cmd_status(args: argparse.Namespace) -> int:
     from java_codebase_rag.jrag_envelope import Envelope
     from java_codebase_rag.jrag_render import render
@@ -1517,7 +1565,7 @@ def _build_node_filter_or_error(filter_dict: dict):
     A bad enum (e.g. ``--role FOO``) should be a user-facing validation error,
     not an internal crash. Returns ``(node_filter, None)`` on success.
     """
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     from java_codebase_rag.jrag_envelope import Envelope
     from pydantic import ValidationError
@@ -1543,7 +1591,7 @@ def _cmd_find_filter_mode(
     limit: int,
 ) -> int:
     """Find filter mode: build NodeFilter and call find_v2."""
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     from java_codebase_rag.jrag_envelope import Envelope, next_actions_hook, normalize_enum, to_envelope_rows
     from java_codebase_rag.jrag_render import render
@@ -1626,7 +1674,7 @@ def _cmd_find_filter_mode(
 
 
 def _cmd_inspect(args: argparse.Namespace) -> int:
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     from java_codebase_rag.jrag_envelope import Envelope, next_actions_hook, resolve_query
     from java_codebase_rag.jrag_render import render
@@ -2526,7 +2574,7 @@ def _cmd_callees(args: argparse.Namespace) -> int:
     # Producer root -> ASYNC_CALLS out (Producer -> :Route, the kafka_topic
     # Route this producer publishes to — NOT a :Producer node).
     if node.kind in ("client", "producer"):
-        import mcp_v2
+        from java_codebase_rag.mcp import mcp_v2
 
         edge_types = ["HTTP_CALLS"] if node.kind == "client" else ["ASYNC_CALLS"]
         out = mcp_v2.neighbors_v2(
@@ -2662,7 +2710,7 @@ def _cmd_callees(args: argparse.Namespace) -> int:
 
 
 def _cmd_hierarchy(args: argparse.Namespace) -> int:
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     cfg, graph, rc = _load_graph_or_error(args)
     if rc:
@@ -2815,7 +2863,7 @@ def _cmd_subclasses(args: argparse.Namespace) -> int:
 
 
 def _cmd_overrides(args: argparse.Namespace) -> int:
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     cfg, graph, rc = _load_graph_or_error(args)
     if rc:
@@ -2868,7 +2916,7 @@ def _cmd_overrides(args: argparse.Namespace) -> int:
 
 
 def _cmd_overridden_by(args: argparse.Namespace) -> int:
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     cfg, graph, rc = _load_graph_or_error(args)
     if rc:
@@ -3176,7 +3224,7 @@ def _cmd_flow(args: argparse.Namespace) -> int:
 
 
 def _cmd_dependencies(args: argparse.Namespace) -> int:
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     cfg, graph, rc = _load_graph_or_error(args)
     if rc:
@@ -3511,7 +3559,7 @@ def _cmd_outline(args: argparse.Namespace) -> int:
     start_line<1). ``--limit`` caps the entry count (the file's symbol table
     is otherwise unbounded); ``truncated`` is set when more entries exist.
     """
-    from ladybug_queries import find_symbols_in_file_range
+    from java_codebase_rag.graph.ladybug_queries import find_symbols_in_file_range
 
     from java_codebase_rag.jrag_envelope import Envelope, mark_truncated, next_actions_hook
     from java_codebase_rag.jrag_render import render
@@ -3581,8 +3629,8 @@ def _cmd_imports(args: argparse.Namespace) -> int:
     a node per import: resolved graph Symbol when resolve_v2 hits (status=one),
     or an unresolved placeholder carrying the raw FQN otherwise.
     """
-    from ast_java import parse_java
-    from resolve_service import resolve_v2
+    from java_codebase_rag.ast.ast_java import parse_java
+    from java_codebase_rag.analysis.resolve_service import resolve_v2
 
     from java_codebase_rag.jrag_envelope import Envelope, next_actions_hook
     from java_codebase_rag.jrag_render import render
@@ -4147,7 +4195,7 @@ def _zero_result_guidance(args: argparse.Namespace, graph) -> str | None:
     empty (truly no matches for this query), or the probe itself errored
     (non-fatal — the empty result still renders).
     """
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
     from collections import Counter
 
     from java_codebase_rag.jrag_envelope import normalize_enum
@@ -4202,7 +4250,7 @@ def _cmd_search(args: argparse.Namespace) -> int:
     truncation, and renders. --fuzzy is rejected IN-HANDLER (not argparse-exit)
     so the error carries the canonical envelope shape.
     """
-    import mcp_v2
+    from java_codebase_rag.mcp import mcp_v2
 
     from java_codebase_rag.jrag_envelope import Envelope, mark_truncated, next_actions_hook, normalize_enum
     from java_codebase_rag.jrag_render import render
@@ -4321,13 +4369,20 @@ def _cmd_search(args: argparse.Namespace) -> int:
             d["kind"] = "search_hit"
         # Add explain token when --explain is set
         if args.explain:
-            from search_lancedb import explain_score_components
+            # search_lancedb is unimportable on graph-only (macOS Intel) installs —
+            # lancedb/sentence-transformers are excluded by the PEP 508 markers in
+            # pyproject.toml, and this module imports them at module top. Importing the
+            # explain renderer from there would crash `jrag search ... --explain` on the
+            # exact Intel install that runs the lexical path. search_scoring is
+            # dependency-free and always installed, so the explain import works everywhere.
+            from java_codebase_rag.search.search_scoring import explain_score_components
             comps = d.get("score_components")
             d["explain"] = explain_score_components(
                 comps,
                 role=d.get("role"),
                 hybrid=bool(args.hybrid),
                 graph_expanded=False,
+                lexical=bool(getattr(out, "lexical_mode", False)),
             )
         hit_dicts.append(d)
 

java_codebase_rag/jrag_envelope.py

@@ -18,7 +18,8 @@ import re
 from dataclasses import dataclass, field
 from typing import Any, Literal
 
-from graph_types import NodeRef
+from java_codebase_rag.absence.absence_types import AbsenceDiagnosis
+from java_codebase_rag.graph.graph_types import NodeRef
 
 __all__ = [
     "Envelope",
@@ -118,6 +119,9 @@ class Envelope:
     # zero in-repo callers. Distinguishes the *correct* empty result ("external
     # entrypoint — no in-repo callers") from a bug-looking bare "0 callers".
     is_external_entrypoint: bool = False
+    # Absence diagnosis explaining why a result is empty (PR-ABS-4). Carried
+    # from MCP outputs and rendered in CLI text/JSON. None on ok/ambiguous.
+    absence: AbsenceDiagnosis | None = None
 
     def to_dict(self) -> dict[str, Any]:
         """Serialize to a JSON-ready dict, omitting empty optionals.
@@ -151,6 +155,8 @@ class Envelope:
             out["message"] = self.message
         if self.is_external_entrypoint:
             out["is_external_entrypoint"] = True
+        if self.absence is not None:
+            out["absence"] = self.absence.model_dump()
         return out
 
     def to_json(self) -> str:
@@ -237,6 +243,8 @@ class Envelope:
             out["message"] = self.message
         if self.is_external_entrypoint:
             out["is_external_entrypoint"] = True
+        if self.absence is not None:
+            out["absence"] = self.absence.model_dump()
         return out
 
     @staticmethod
@@ -524,10 +532,10 @@ def resolve_query(
     means ``--service`` disambiguates which microservice's route is selected.
     """
     # Lazy imports — keeps build_parser() / `jrag --help` free of resolve/ladybug.
-    from resolve_service import resolve_v2
+    from java_codebase_rag.analysis.resolve_service import resolve_v2
 
     if graph is None:
-        from ladybug_queries import LadybugGraph
+        from java_codebase_rag.graph.ladybug_queries import LadybugGraph
 
         graph = LadybugGraph.get(str(cfg.ladybug_path))
 
@@ -610,7 +618,7 @@ def resolve_query(
     # the agent-facing CLI).
     if "jrag search" not in raw_msg:
         raw_msg = f"{raw_msg} Use `jrag search <query>` for ranked fuzzy lookup."
-    return None, Envelope(status="not_found", message=raw_msg)
+    return None, Envelope(status="not_found", message=raw_msg, absence=out.absence)
 
 
 # Listing breadcrumbs (root is None): 1–2 template hints pointing at the natural
@@ -1082,4 +1090,5 @@ def project_envelope(envelope: Envelope, detail: str) -> Envelope:
         file_location=envelope.file_location,
         message=envelope.message,
         is_external_entrypoint=envelope.is_external_entrypoint,
+        absence=envelope.absence,
     )

java_codebase_rag/jrag_hints.py

@@ -143,7 +143,7 @@ def next_actions(
     # EDGE_SCHEMA is the canonical label set; we use it to skip labels we don't
     # recognize (avoids emitting hints for spurious / future edge types the
     # command map doesn't cover).
-    from java_ontology import EDGE_SCHEMA
+    from java_codebase_rag.graph.java_ontology import EDGE_SCHEMA
 
     # Known virtual labels not in EDGE_SCHEMA (describe-time rollup constructs).
     _VIRTUAL_LABELS = frozenset({"OVERRIDDEN_BY"})

java_codebase_rag/jrag_render.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 from typing import Any
 
+from java_codebase_rag.absence.absence_types import AbsenceDiagnosis
 from java_codebase_rag.jrag_envelope import Envelope, project_envelope, simple_name
 
 __all__ = ["render", "tiered_name", "display_name"]
@@ -32,6 +33,23 @@ _CALLS_FAMILY_EDGES = frozenset({"CALLS", "HTTP_CALLS", "ASYNC_CALLS"})
 # none) and are left untagged.
 _ROUTE_KIND_TAGS: dict[str, str] = {"kafka_topic": "kafka", "http_endpoint": "http"}
 
+# Absence verdict → human-readable label, shared by the not-found / listing /
+# traversal empty-result renderers. ``AbsenceVerdict`` is a closed Literal of
+# these four values.
+_ABSENCE_VERDICT_TEXT: dict[str, str] = {
+    "not_in_project": "not in project",
+    "external_dependency": "external dependency",
+    "refine_query": "refine your query",
+    "correct_empty": "correct empty",
+}
+
+
+def _verdict_line(absence: AbsenceDiagnosis) -> str | None:
+    """A ``Verdict: <label>`` line for an absence diagnosis, or ``None`` if the
+    verdict is not one of the known values."""
+    text = _ABSENCE_VERDICT_TEXT.get(absence.verdict)
+    return f"Verdict: {text}" if text else None
+
 # Identity keys already represented in a listing line (display_name + @service +
 # kind tag). At ``--detail full`` the per-row kv-block skips these (they are in
 # the header line) and renders every OTHER key, so full listing == per-row
@@ -226,7 +244,35 @@ def _render_error(envelope: Envelope) -> str:
 
 def _render_not_found(envelope: Envelope) -> str:
     msg = envelope.message or "not found"
-    return f"not found: {msg}"
+    base = f"not found: {msg}"
+
+    # If absence diagnosis is present, append verdict + message (+ did-you-mean)
+    if envelope.absence is not None:
+        lines = [base]
+        # Verdict line (human-readable label)
+        vline = _verdict_line(envelope.absence)
+        if vline:
+            lines.append(vline)
+
+        # Per-cause explanation message — surfaces the diagnosis's authored help
+        # (external-identity context, filter-relaxation suggestions, etc.).
+        if envelope.absence.message:
+            lines.append(envelope.absence.message)
+
+        # Add did-you-mean line if closest_symbols is non-empty
+        if envelope.absence.closest_symbols:
+            symbols = [s.fqn for s in envelope.absence.closest_symbols]
+            if len(symbols) == 1:
+                lines.append(f"Did you mean: {symbols[0]}?")
+            elif len(symbols) == 2:
+                lines.append(f"Did you mean: {symbols[0]} or {symbols[1]}?")
+            else:
+                joined = ", ".join(symbols[:-1]) + f", or {symbols[-1]}"
+                lines.append(f"Did you mean: {joined}?")
+
+        return "\n".join(lines)
+
+    return base
 
 
 def _render_listing(envelope: Envelope, *, noun: str, detail: str = "normal") -> str:
@@ -295,7 +341,12 @@ def _render_listing(envelope: Envelope, *, noun: str, detail: str = "normal") ->
             if rest:
                 lines.extend(_render_inspect_block(rest, 1))
     if not lines:
-        lines.append(f"0 {noun}".rstrip())
+        # Handle absence diagnosis (PR-ABS-4)
+        if envelope.absence is not None:
+            vline = _verdict_line(envelope.absence)
+            lines.append(vline if vline else f"0 {noun}".rstrip())
+        else:
+            lines.append(f"0 {noun}".rstrip())
     # Listing breadcrumbs (Phase 2): <=2 `next:` hint lines when the listing
     # command emitted agent_next_actions (routes/clients/producers/topics).
     lines.extend(_next_action_lines(envelope))
@@ -413,10 +464,23 @@ def _render_traversal(envelope: Envelope, *, noun: str, detail: str = "normal")
         root_node = envelope.nodes.get(root_id, {})
         root_fqn = str(root_node.get("fqn") or "").strip()
         root_svc = str(root_node.get("microservice") or "").strip()
-        if envelope.is_external_entrypoint:
+
+        # Handle absence diagnosis (PR-ABS-4)
+        if envelope.absence is not None:
+            absence = envelope.absence
+            if absence.verdict == "correct_empty":
+                # Same text as the is_external_entrypoint case.
+                parts = ["external entrypoint — no in-repo callers"]
+            else:
+                vline = _verdict_line(absence)
+                if vline:
+                    lines.append(vline)
+                parts = [f"0 {noun}".rstrip()]
+        elif envelope.is_external_entrypoint:
             parts = ["external entrypoint — no in-repo callers"]
         else:
             parts = [f"0 {noun}".rstrip()]
+
         if root_fqn:
             parts.append(root_fqn)
         if root_svc:

mcp_hints.py → java_codebase_rag/mcp/mcp_hints.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import json
 from typing import Any, Literal, NamedTuple
 
-from java_ontology import EDGE_SCHEMA, FUZZY_STRATEGY_SET
+from java_codebase_rag.graph.java_ontology import EDGE_SCHEMA, FUZZY_STRATEGY_SET
 
 # Normative schema description (propose §3.1) — imported by ``mcp_v2`` for Field(description=...).
 MCP_HINTS_STRUCTURED_FIELD_DESCRIPTION = (