java-codebase-rag 0.6.7__py3-none-any.whl → 0.9.0__py3-none-any.whl

search_lancedb.py

@@ -41,6 +41,11 @@ JAVA_ENRICHED_COLUMNS: tuple[str, ...] = (
     "capabilities",
 )
 
+# Over-fetch multiplier for dedup: fetch 4x to absorb per-FQN chunk multiplicity
+# so that after collapsing by primary_type_fqn, a page stays full and the +1
+# truncation sentinel survives. The formula: need = max((limit + offset) * 4, limit + offset + 1)
+DEDUP_OVERFETCH = 4
+
 VECTOR_COLUMN = "embedding"
 _FTS_READY: set[tuple[str, str]] = set()
 _FTS_LOCK = threading.Lock()
@@ -201,6 +206,14 @@ _ROLE_SCORE_WEIGHTS: dict[str, float] = {
     "DTO": -0.08,
 }
 
+# Theoretical maximum for hybrid composite score (used for display normalization).
+# Hybrid sort metric: raw_rrf * (import_factor if import_heavy else 1)
+#                   + role_weight + symbol_bonus
+# where raw_rrf ≤ 2/(k+1) for 2-list RRF, role_weight ≤ max(_ROLE_SCORE_WEIGHTS),
+# and symbol_bonus ≤ _SYMBOL_MATCH_BONUS_CAP + _TYPE_MATCH_BONUS_CAP + _ACTION_VERB_BONUS.
+# The import factor is ≤ 1, so we use the raw max (2/61).
+_HYBRID_SCORE_MAX = (2.0 / 61.0) + max(_ROLE_SCORE_WEIGHTS.values()) + _SYMBOL_MATCH_BONUS_CAP + _TYPE_MATCH_BONUS_CAP + _ACTION_VERB_BONUS
+
 
 def _query_tokens(query: str) -> set[str]:
     """Lowercased alpha-only tokens from the query, minus stopwords, len >= 3.
@@ -353,7 +366,7 @@ def _hybrid_sort_key(r: dict) -> float:
     comps["hybrid_rrf"] = s
     if r.get("_hints", {}).get("import_heavy"):
         s *= _IMPORT_HYBRID_SCORE_FACTOR
-        comps["import_penalty"] = _IMPORT_HYBRID_SCORE_FACTOR
+        comps["import_penalty"] = 1.0 - _IMPORT_HYBRID_SCORE_FACTOR
     s += _role_weight(r)
     s += float(comps.get("symbol_bonus", 0.0))
     return -s
@@ -376,7 +389,8 @@ def explain_score_components(
         comps = {}
     parts: list[str] = []
     if hybrid:
-        rrf = comps.get("hybrid_rrf")
+        # Prefer rrf_raw (added by PR-SEARCH-1a) for explanation
+        rrf = comps.get("rrf_raw") or comps.get("hybrid_rrf")
         if rrf is not None:
             parts.append(f"rrf={float(rrf):.3f}")
     else:
@@ -403,6 +417,46 @@ def l2_distance_to_score(distance: float) -> float:
     return 1.0 - distance * distance / 2.0
 
 
+def _effective_distance(comps: dict[str, float]) -> float:
+    """Compute the adjusted distance used for sorting.
+
+    Matches _vector_sort_key logic: distance + import_penalty - role_weight - symbol_bonus.
+    """
+    d = comps.get("distance", 0.0)
+    d += comps.get("import_penalty", 0.0)
+    d -= comps.get("role_weight", 0.0)
+    d -= comps.get("symbol_bonus", 0.0)
+    return d
+
+
+def _clamp01(x: float) -> float:
+    """Clamp a value to the [0.0, 1.0] range."""
+    if x < 0.0:
+        return 0.0
+    if x > 1.0:
+        return 1.0
+    return x
+
+
+def _hybrid_post_sort_normalization(rows: list[dict]) -> None:
+    """Set honest displayed scores for hybrid search after sorting.
+
+    Reconstructs the composite score (raw_rrf * import_factor + role_weight + symbol_bonus)
+    and normalizes by _HYBRID_SCORE_MAX to ensure rank-monotonicity.
+
+    Mutates rows in-place, replacing _score with the normalized value.
+    """
+    for r in rows:
+        comps = r.setdefault("_score_components", {})
+        raw = comps.get("hybrid_rrf", 0.0)
+        comps["rrf_raw"] = raw  # preserve raw RRF for --explain. NOTE: when graph_expand + hybrid combine (Phase 2), _rrf_merge below overwrites this with graph-RRF, so --explain would show graph-RRF not hybrid-RRF.
+        s = raw
+        if r.get("_hints", {}).get("import_heavy"):
+            s *= _IMPORT_HYBRID_SCORE_FACTOR
+        s += comps.get("role_weight", 0.0) + comps.get("symbol_bonus", 0.0)
+        r["_score"] = _clamp01(s / _HYBRID_SCORE_MAX)
+
+
 def _escape_like_fragment(s: str) -> str:
     return s.replace("'", "''")
 
@@ -534,6 +588,15 @@ def _search_one_table(
     for r in rows:
         r["_kind"] = kind
         r["_hybrid"] = False
+        # Populate `_score` from `_distance` so the SearchHit.score reflects
+        # relevance. The hybrid branch sets `_score` from `_relevance_score`
+        # above; without this, non-hybrid (default) search left `_score` unset
+        # and mcp_v2._row_to_search_hit fell back to 0.0 for EVERY hit —
+        # ranking still worked (the sort key uses `_distance` directly) but the
+        # exposed score was always 0.0, making results look unranked.
+        d = r.get("_distance")
+        if d is not None:
+            r["_score"] = l2_distance_to_score(float(d))
         r["start"] = coerce_position_field(r.get("start"))
         r["end"] = coerce_position_field(r.get("end"))
     return rows
@@ -781,9 +844,72 @@ def _rrf_merge(
                 existing["_rrf_score"] = float(existing.get("_rrf_score", 0.0)) + contribution
     merged = list(pool.values())
     merged.sort(key=lambda r: -float(r.get("_rrf_score", 0.0)))
+    # Normalize displayed _rrf_score to [0,1] by theoretical max
+    # RRF max = Σ weight·1/(k+rank+1); theoretical max when all rows are rank 0
+    # with weight 1.0 = num_lists / (k + 1)
+    num_lists = len(lists)
+    max_rrf = num_lists / (k + 1)
+    for r in merged:
+        raw_score = float(r.get("_rrf_score", 0.0))
+        comps = r.setdefault("_score_components", {})
+        comps["rrf_raw"] = raw_score
+        r["_rrf_score"] = _clamp01(raw_score / max_rrf)
     return merged
 
 
+def _dedup_by_fqn(rows: list[dict], dedup_by_fqn: bool = True) -> list[dict]:
+    """Deduplicate rows by primary_type_fqn (java table only).
+
+    When dedup_by_fqn is True, collapses multiple chunks of the same
+    primary_type_fqn into one row (first-seen-wins, since rows are pre-sorted
+    so the first is the best chunk). Each survivor gets a _chunks_collapsed
+    field (>=1) counting how many rows were collapsed into it.
+
+    Rows without primary_type_fqn (sql/yaml tables) get a unique __id:<id>
+    key so they pass through unchanged (each row is unique).
+
+    When dedup_by_fqn is False, returns rows unchanged (regression guard).
+    """
+    if not dedup_by_fqn:
+        # Non-dedup path: return unchanged, byte-identical to prior behavior
+        return rows
+
+    deduped: list[dict] = []
+    seen_keys: dict[str, dict] = {}
+    collapsed_counts: dict[str, int] = {}
+
+    for row in rows:
+        # Build dedup key: primary_type_fqn for java rows, unique __id:<id> for sql/yaml
+        fqn = row.get("primary_type_fqn")
+        if fqn:
+            key = str(fqn)
+        else:
+            # sql/yaml rows have no primary_type_fqn → unique key per row
+            row_id = row.get("id") or id(row)
+            key = f"__id:{row_id}"
+
+        if key not in seen_keys:
+            # First occurrence: keep it
+            seen_keys[key] = row
+            collapsed_counts[key] = 1
+            deduped.append(row)
+        else:
+            # Duplicate: increment collapse count, discard this row
+            collapsed_counts[key] += 1
+
+    # Annotate each survivor with _chunks_collapsed
+    for row in deduped:
+        fqn = row.get("primary_type_fqn")
+        if fqn:
+            key = str(fqn)
+        else:
+            row_id = row.get("id") or id(row)
+            key = f"__id:{row_id}"
+        row["_chunks_collapsed"] = collapsed_counts[key]
+
+    return deduped
+
+
 def run_search(
     query: str,
     *,
@@ -810,6 +936,7 @@ def run_search(
     exclude_roles: list[str] | None = None,
     capability: str | None = None,
     capability_in: list[str] | None = None,
+    dedup_by_fqn: bool = False,
 ) -> list[dict]:
     effective_hybrid = hybrid
     effective_fts = fts_text
@@ -843,7 +970,16 @@ def run_search(
     fts_for_hybrid = effective_fts if effective_fts is not None else query
 
     db = lancedb.connect(uri)
-    need = max(limit + offset, 1)
+    if dedup_by_fqn:
+        # Over-fetch to absorb per-FQN chunk multiplicity: fetch 4x so that
+        # after collapsing, the page stays full and the +1 truncation sentinel survives.
+        # The 4× factor assumes typical per-FQN chunk multiplicity; a single type with
+        # many high-ranking chunks (e.g. generated/God classes) could starve the page or
+        # make the +1 truncation sentinel unreliable; Phase 1 may revisit adaptive over-fetch (plan risk #1).
+        need = max((limit + offset) * DEDUP_OVERFETCH, limit + offset + 1)
+    else:
+        # Non-dedup path: exact fetch as before
+        need = max(limit + offset, 1)
 
     extra_java = _build_extra_predicates(
         columns=_table_columns(uri, TABLES["java"], db),
@@ -853,7 +989,7 @@ def run_search(
         capability=capability, capability_in=capability_in,
     ) if "java" in table_keys else []
 
-    skip_role_weight = bool(role or role_in)
+    skip_role_weight = bool(role or role_in or exclude_roles)
     query_toks = _query_tokens(query)
 
     if len(table_keys) == 1:
@@ -878,8 +1014,15 @@ def run_search(
         _apply_symbol_bonus(rows, query_toks)
         if effective_hybrid:
             rows.sort(key=_hybrid_sort_key)
+            # Hybrid: set honest displayed score from composite sort metric, clamped to [0,1]
+            _hybrid_post_sort_normalization(rows)
         else:
             rows.sort(key=_vector_sort_key)
+            # Vector: set honest displayed score from adjusted distance, clamped to [0,1]
+            for r in rows:
+                comps = r.setdefault("_score_components", {})
+                effective_dist = _effective_distance(comps)
+                r["_score"] = _clamp01(l2_distance_to_score(effective_dist))
 
         if graph_expand and key == "java" and expand_depth > 0:
             rows = _graph_expand_merge(
@@ -893,6 +1036,9 @@ def run_search(
                 ladybug_path=ladybug_path,
             )
 
+        # Dedup by primary_type_fqn after all sorting/merging, before windowing
+        rows = _dedup_by_fqn(rows, dedup_by_fqn=dedup_by_fqn)
+
         window = rows[offset : offset + limit]
         if context_neighbors > 0 and key == "java":
             _attach_neighbor_context(window, db=db, neighbors=context_neighbors, uri=uri)
@@ -922,6 +1068,15 @@ def run_search(
             r["_skip_role_weight"] = True
     _apply_symbol_bonus(merged, query_toks)
     merged.sort(key=_vector_sort_key)
+    # Vector: set honest displayed score from adjusted distance, clamped to [0,1]
+    for r in merged:
+        comps = r.setdefault("_score_components", {})
+        effective_dist = _effective_distance(comps)
+        r["_score"] = _clamp01(l2_distance_to_score(effective_dist))
+
+    # Dedup by primary_type_fqn after all sorting/merging, before windowing
+    merged = _dedup_by_fqn(merged, dedup_by_fqn=dedup_by_fqn)
+
     window = merged[offset : offset + limit]
     if context_neighbors > 0:
         _attach_neighbor_context(window, db=db, neighbors=context_neighbors, uri=uri)

server.py

@@ -26,7 +26,8 @@ from java_codebase_rag.config import (
 from ladybug_queries import LadybugGraph, resolve_ladybug_path
 from mcp.server.fastmcp import FastMCP
 from pydantic import BaseModel, Field
-from search_lancedb import TABLES
+# NOTE: search_lancedb.TABLES is imported lazily in list_code_index_tables_payload() — it
+# pulls lancedb/torch and is unavailable on graph-only installs (macOS Intel).
 
 _COCOINDEX_TARGET = "java_index_flow_lancedb.py:JavaCodeIndexLance"
 _INSTRUCTIONS = (
@@ -296,12 +297,19 @@ def _graph_meta_output() -> GraphMetaOutput:
 
 
 def list_code_index_tables_payload() -> IndexInfoOutput:
+    try:
+        from search_lancedb import TABLES
+
+        tables = dict(TABLES)
+    except ImportError:
+        # Graph-only install (no lancedb): no Lance vector tables exist.
+        tables = {}
     return IndexInfoOutput(
         lancedb_uri=_resolve_lancedb_uri(),
         embedding_model=resolved_sbert_model_for_process_env(SBERT_MODEL),
         project_root=str(_project_root()),
         cocoindex_target=_COCOINDEX_TARGET,
-        tables=dict(TABLES),
+        tables=tables,
         graph=_graph_meta_output(),
     )
 
@@ -499,11 +507,12 @@ def create_mcp_server() -> FastMCP:
             "Ranked chunk retrieval over content tables (java/sql/yaml); `query` is opaque text (natural language or code "
             "fragments) and results are score-ranked, not boolean-matched. For graph-structured listing "
             "(symbols/routes/clients/producers) use `find`, not `search`. Optional `filter` uses the same NodeFilter "
-            "schema as `find` but only **symbol-applicable** fields apply — others return success=false. Wildcards "
-            "(`*`, `?`) in prefix fields are rejected—use ranked `query` text instead. There is **no** "
+            "schema as `find` but only **symbol-applicable** fields apply — others return success=false. Substring "
+            "fields match literally (no `*`/`?` metacharacters)—use ranked `query` text for fuzzy discovery. There is **no** "
             "structured DSL inside `query`; structured predicates belong in `find`. "
-            "For identifier-shaped lookups (FQN, id prefix, route/client identifiers, …), use `resolve` first; "
+            "For identifier-shaped lookups (FQN, id, route/client identifiers, …), use `resolve` first; "
             "use `search` for natural-language or ranked fuzzy discovery. "
+            "Set `explain=true` to include score breakdown per hit. "
             "Successful responses echo `limit`/`offset`."
         ),
     )
@@ -530,6 +539,14 @@ def create_mcp_server() -> FastMCP:
                 "predicate. Unknown keys or populated fields not applicable to symbols return success=false."
             ),
         ),
+        explain: bool = Field(
+            default=False,
+            description="If true, include score_components in each SearchHit (breakdown of distance/rrf, role, symbol, import_penalty).",
+        ),
+        chunks: bool = Field(
+            default=False,
+            description="If true, show every chunk (default collapses to one row per symbol/type).",
+        ),
     ) -> mcp_v2.SearchOutput:
         scoped_filter = _scope_manager.apply_auto_scope(filter) if _scope_manager else filter
         return await asyncio.to_thread(
@@ -541,20 +558,22 @@ def create_mcp_server() -> FastMCP:
             offset,
             path_contains,
             scoped_filter,
+            explain,
             None,
+            not chunks,  # dedup=True by default; chunks=True opts out
         )
 
     @mcp.tool(
         name="find",
         description=(
             "Exact structured listing for one node kind. Per-kind applicable fields: **symbol** — "
-            "microservice, module, role, exclude_roles, annotation, capability, fqn_prefix, symbol_kind, symbol_kinds; "
-            "**route** — microservice, module, http_method, path_prefix, framework; **client** — microservice, module, "
-            "source_layer, client_kind, target_service, target_path_prefix, http_method; **producer** — microservice, "
-            "module, source_layer, producer_kind, topic_prefix. "
+            "microservice, module, role, exclude_roles, annotation, capability, fqn_contains, symbol_kind, symbol_kinds; "
+            "**route** — microservice, module, http_method, path_contains, framework; **client** — microservice, module, "
+            "source_layer, client_kind, target_service, target_path_contains, http_method; **producer** — microservice, "
+            "module, source_layer, producer_kind, topic_contains. "
             "`role` is singular and `exclude_roles` plural; `capability` is a functional tag assigned during indexing. "
-            "`fqn_prefix` is a prefix predicate — for exact FQN or id lookup use `resolve`/`describe`. "
-            "Wildcards in prefix fields are rejected. An empty filter (`{}`) or `filter=None` means no predicate (all nodes of "
+            "`fqn_contains` is a substring predicate — for exact FQN or id lookup use `resolve`/`describe`. "
+            "Substring fields match literally (Cypher `CONTAINS`); no wildcard metacharacters. An empty filter (`{}`) or `filter=None` means no predicate (all nodes of "
             "that kind; use pagination). Unknown keys or inapplicable populated fields return success=false. "
             "Successful responses echo `limit`/`offset`."
         ),
@@ -620,7 +639,7 @@ def create_mcp_server() -> FastMCP:
             "`direction` and `edge_types` have no defaults; an empty `edge_types` fails. The CALLS-only features — "
             "`edge_filter`, `include_unresolved`, `dedup_calls` — each require `edge_types=['CALLS']`; `edge_filter` and "
             "`include_unresolved` are mutually exclusive. Violating a precondition (wrong CALLS context, composed/override "
-            "keys on an ineligible origin or with `direction='in'`, wildcards in prefix fields, unknown filter keys) returns "
+            "keys on an ineligible origin or with `direction='in'`, unknown filter keys) returns "
             "success=false with a message; `dedup_calls` with other edge_types is a silent no-op. "
             "Optional `filter` applies to each neighbor endpoint row; populated fields must be applicable to that "
             "neighbor's kind—mixed-kind result sets fail on the first inapplicable neighbor (per-neighbor strict frame). "

java_codebase_rag-0.6.7.dist-info/RECORD

@@ -1,34 +0,0 @@
-ast_java.py,sha256=Paee3ZV9G5iy8LqfkVq0Ah_1fV0k632oS5JPkiWzwB8,99206
-brownfield_events.py,sha256=yxXkKDgMb3VPtaiakGzncHM_EGnda8xIue6w90yYp8s,2055
-build_ast_graph.py,sha256=jubb9Ex_6B3ktsInqNkk-EksBtQjofoFhE92lmHr308,169659
-chunk_heuristics.py,sha256=aQk2NOKxzUdqoUAJUO3G3LE0MN_bYZWNLQ0tkmj5uts,1813
-graph_enrich.py,sha256=W_OQK7YRU1K8wi-vfrxZWUd-EErTqWx840UBrhVpNsM,62788
-index_common.py,sha256=HT6FKHFJ084eFvd3fR1j8z8gf4eWoPHVW8GXLpw464I,285
-java_index_flow_lancedb.py,sha256=JHdRWDZoZ3wnxi8NNG2ck8zEmSALHB2UVB1N8TSvlmg,24109
-java_index_v1_common.py,sha256=nF1KrSqboF_RRvWerG9knRRFmWwsrG_CvhgnsoZ8KqA,1154
-java_ontology.py,sha256=71bCLDNvMy0SpZPzSR5apJ0qJXNd6y5ggkLdBEw_PFo,16682
-ladybug_queries.py,sha256=7vSP7WsUKQYXHFenBMlCdpUFed39h3oul-WRkf55YVc,90330
-mcp_hints.py,sha256=3swh05LSiWur3tm3-yssndBsLxIxFhy501kBtJI8jJ0,42509
-mcp_v2.py,sha256=S8PiVGDlyI7cvTmeMdQobhKUxWqKoY2YxXXr4pu9lQI,80348
-path_filtering.py,sha256=R--XzI51LXBu5IBKMCnJWbkNr6I5d-SDmltyQQnWco0,17674
-pr_analysis.py,sha256=zrmZZD5yotJtM02Kif6_jgI_oeformOao793akp0N6Y,18394
-search_lancedb.py,sha256=ga_qySeCzA6hCibxUPXpCbTxW9mCjTdwirMhMhfNCz4,36801
-server.py,sha256=DcJwocSknBy0vwsUvYiC5hr-hrD_rRT-u9_DmuRTC9k,35035
-java_codebase_rag/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-java_codebase_rag/_fdlimit.py,sha256=WroFdfSNbcriKok6q8znTf74dqlznxea_1Fd5bHl_3o,1930
-java_codebase_rag/cli.py,sha256=zMZcjVKRgfv7VpTb0h1vjGGzPXY4NT2Uqyz77uysc0c,40085
-java_codebase_rag/cli_format.py,sha256=CT7-xdwZ0bMCdP68_UOwkvm-mnLluU3LutlM-mDNk60,1839
-java_codebase_rag/cli_progress.py,sha256=q6Wh97yzLGs1B8UFk_WAKivfQu7Y5RnUUE-T2YHWkIs,3237
-java_codebase_rag/config.py,sha256=bfwYI4R8PU9YV_M4r8-03iaUZ_0TW-qN_NuhIsDXy2M,18769
-java_codebase_rag/installer.py,sha256=sXsHPo24aoDFoTr0D_vYLg0MFdGAV2wdL05FqRaul6E,52861
-java_codebase_rag/lance_optimize.py,sha256=25Rwj7HNO8F-35MxhFK6naqgbjd3H-T0zKb3pXB4H0s,9268
-java_codebase_rag/pipeline.py,sha256=ydNktEGL1YniAjJsr37yKBo_bGV4cN_LTGVTmmrsrZw,14688
-java_codebase_rag/progress.py,sha256=2IxdMALDM0wAQCyJrrfZ975zM_85C-4BfHxf4AtYifE,23212
-java_codebase_rag/install_data/agents/explorer-rag-enhanced.md,sha256=BkdQpBEWqSdvGHgbqMdRb5CWfEiFRJK4Dgqbyal3l6s,14551
-java_codebase_rag/install_data/skills/explore-codebase/SKILL.md,sha256=YkRnrM7Wh5E8raFjAW3RrN2V9-ov8upaGC3UdpSx6U8,12346
-java_codebase_rag-0.6.7.dist-info/licenses/LICENSE,sha256=gxvtiHtuviR_q8ZAjWw-QTcF3DyPzg6ZY-lQrr8OPpw,1068
-java_codebase_rag-0.6.7.dist-info/METADATA,sha256=Hnwc5Zpo53La7PXgldTTlVl87xRGG5KvDTky6ufxyQM,17239
-java_codebase_rag-0.6.7.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-java_codebase_rag-0.6.7.dist-info/entry_points.txt,sha256=wsPZwot0Ui4JI3TIgW8LcbN8bNtKFbwQAlHAAJXfYgQ,117
-java_codebase_rag-0.6.7.dist-info/top_level.txt,sha256=syQgi8XPBwY2ws_NZ1uRCxTf_s41NpshwEHNdcdnk3A,245
-java_codebase_rag-0.6.7.dist-info/RECORD,,