java-codebase-rag 0.11.2__py3-none-any.whl → 0.12.1__py3-none-any.whl

java_codebase_rag/eval/__init__.py

@@ -1 +0,0 @@
-# Eval package for IR metrics

java_codebase_rag/eval/ground_truth.py

@@ -1,100 +0,0 @@
-"""Eval ground-truth — Tier-A auto generator + Tier-B file loader.
-
-Tier-A derives labeled queries deterministically from indexed symbols (no
-manual labeling). Tier-B loads hand-curated labeled queries from YAML/JSON.
-"""
-
-from __future__ import annotations
-
-import json
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Iterable, Protocol
-
-import yaml
-
-from java_codebase_rag.search.search_scoring import _split_identifier
-
-
-class SymbolLike(Protocol):
-    """Structural type for symbols — duck-typed .fqn / .name."""
-
-    fqn: str
-    name: str
-
-
-@dataclass(frozen=True)
-class LabeledQuery:
-    """A labeled retrieval query and its set of relevant Symbol FQNs."""
-
-    query: str
-    relevant: frozenset[str]
-    tier: str
-
-
-def build_tier_a(symbols: Iterable[SymbolLike]) -> list[LabeledQuery]:
-    """Auto-generate labeled queries from each symbol's simple name.
-
-    For each symbol two query strings are derived from its simple name:
-      1. The original simple name verbatim (e.g. ``"DistributionChunkService"``)
-         — matches identifier-joined index text.
-      2. A space-joined lowercase token form (e.g. ``"distribution chunk service"``)
-         produced via ``search_scoring._split_identifier`` so tokenization parity
-         with the FTS index holds.
-
-    Symbols whose simple name splits to fewer than 2 tokens or is shorter than
-    3 characters are skipped (noise). Output is deterministic, sorted by
-    ``(query, fqn)``; all entries carry ``tier="A"``.
-    """
-    out: list[LabeledQuery] = []
-    for sym in symbols:
-        name: str = sym.name
-        if len(name) < 3:
-            continue
-        tokens = _split_identifier(name)
-        if len(tokens) < 2:
-            continue
-        fqn: str = sym.fqn
-        relevant = frozenset({fqn})
-        # 1. identifier-joined form = ORIGINAL simple name (preserve case).
-        out.append(LabeledQuery(name, relevant, "A"))
-        # 2. space-joined lowercase token form.
-        out.append(LabeledQuery(" ".join(tokens), relevant, "A"))
-    out.sort(key=lambda q: (q.query, next(iter(q.relevant))))
-    return out
-
-
-def load_tier_b(path: str | Path) -> list[LabeledQuery]:
-    """Load hand-curated Tier-B labeled queries from a YAML (``.yaml``/``.yml``)
-    or JSON (``.json``) file.
-
-    Schema: a list of ``{query: str, relevant: [str, ...]}`` objects.
-
-    Raises:
-        FileNotFoundError: if the path does not exist (the runner checks
-            existence before calling, treating absence as "Tier-B disabled").
-    """
-    p = Path(path)
-    if not p.exists():
-        raise FileNotFoundError(f"Tier-B ground-truth file not found: {p}")
-
-    suffix = p.suffix.lower()
-    raw = p.read_text()
-    if suffix in (".yaml", ".yml"):
-        data = yaml.safe_load(raw)
-    elif suffix == ".json":
-        data = json.loads(raw)
-    else:
-        # Fall back to YAML (superset of JSON) for unknown extensions.
-        data = yaml.safe_load(raw)
-
-    out: list[LabeledQuery] = []
-    for entry in data or []:
-        out.append(
-            LabeledQuery(
-                query=str(entry["query"]),
-                relevant=frozenset(entry.get("relevant", []) or []),
-                tier="B",
-            )
-        )
-    return out

java_codebase_rag/eval/metrics.py

@@ -1,107 +0,0 @@
-"""IR evaluation metrics — pure functions, stdlib only.
-
-Functions take `retrieved: list[str]` (ordered list of retrieved FQN ids)
-and `relevant: set[str]` (ground-truth relevant set).
-"""
-
-from __future__ import annotations
-
-
-def recall_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
-    """Fraction of relevant documents appearing in retrieved[:k].
-
-    Args:
-        retrieved: Ordered list of retrieved document IDs.
-        relevant: Set of ground-truth relevant document IDs.
-        k: Cut-off rank (1-indexed).
-
-    Returns:
-        Recall@k in [0.0, 1.0]. Returns 0.0 if relevant is empty.
-    """
-    if not relevant:
-        return 0.0
-
-    retrieved_at_k = set(retrieved[:k])
-    relevant_retrieved = retrieved_at_k.intersection(relevant)
-
-    return len(relevant_retrieved) / len(relevant)
-
-
-def precision_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
-    """Precision at cut-off k: |retrieved[:k] ∩ relevant| / k.
-
-    Args:
-        retrieved: Ordered list of retrieved document IDs.
-        relevant: Set of ground-truth relevant document IDs.
-        k: Cut-off rank (1-indexed).
-
-    Returns:
-        Precision@k in [0.0, 1.0]. Returns 0.0 if k == 0.
-    """
-    if k == 0:
-        return 0.0
-
-    retrieved_at_k = set(retrieved[:k])
-    relevant_retrieved = retrieved_at_k.intersection(relevant)
-
-    return len(relevant_retrieved) / k
-
-
-def reciprocal_rank(retrieved: list[str], relevant: set[str]) -> float:
-    """Reciprocal rank: 1.0 / rank of first retrieved relevant document.
-
-    Args:
-        retrieved: Ordered list of retrieved document IDs.
-        relevant: Set of ground-truth relevant document IDs.
-
-    Returns:
-        Reciprocal rank in [0.0, 1.0]. Returns 0.0 if no relevant document
-        is retrieved. Ranks are 1-indexed (first hit = 1.0).
-    """
-    for rank, doc_id in enumerate(retrieved, start=1):
-        if doc_id in relevant:
-            return 1.0 / rank
-
-    return 0.0
-
-
-def mean(values: list[float]) -> float:
-    """Arithmetic mean of a list of floats.
-
-    Args:
-        values: List of float values.
-
-    Returns:
-        Arithmetic mean. Returns 0.0 for empty list.
-    """
-    if not values:
-        return 0.0
-
-    return sum(values) / len(values)
-
-
-def aggregate(per_query: list[dict]) -> dict[str, float]:
-    """Aggregate per-query metrics into means across all queries.
-
-    Takes a list of per-query metric dicts and computes the mean of each
-    metric across all queries. All dicts must have the same keys.
-
-    Args:
-        per_query: List of dicts, each containing metric names as keys
-            and float values (e.g., {"recall@10": 1.0, "mrr": 0.5, ...}).
-
-    Returns:
-        Dict with same keys as input, containing mean of each metric
-        across all queries (e.g., {"recall@10": 0.42, "mrr": 0.55, ...}).
-    """
-    if not per_query:
-        return {}
-
-    metric_names = list(per_query[0].keys())
-    result: dict[str, float] = {}
-
-    for metric in metric_names:
-        values = [query_metrics[metric] for query_metrics in per_query]
-        result[metric] = mean(values)
-
-    return result