deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/eval/matching.py

@@ -0,0 +1,218 @@
+"""Gold-passage matching for retrieval evaluation.
+
+An engine returns ranked chunks, not ground-truth identifiers, so relevance has
+to be decided from text. A chunk counts as covering a gold passage when either
+
+* one normalized text contains the other — the common cases are a chunk holding
+  the excerpt verbatim, and a *truncated* citation whose opening text sits inside
+  a longer gold passage (LlamaIndex and GraphRAG both cap ``content`` at 200
+  characters); or
+* the two share enough tokens — this catches a paraphrased or synthesized
+  citation (LightRAG reports, GraphRAG community summaries) that never repeats
+  the excerpt word for word.
+
+The token rule needs both a coverage ratio and a minimum shared-token count, so a
+short generic phrase ("the model") cannot claim a long passage it merely happens
+to appear in. Tokens are ASCII words plus individual CJK characters, which keeps
+a Chinese excerpt comparable without a segmenter dependency.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+import re
+from typing import Any
+
+#: Share of the shorter side's tokens that must overlap for an overlap match.
+DEFAULT_MIN_RATIO = 0.5
+#: Shared tokens required before an overlap match is accepted.
+DEFAULT_MIN_TOKENS = 4
+
+_WHITESPACE = re.compile(r"\s+")
+# ASCII words and digits, or one token per CJK / Kana / Hangul character.
+_TOKEN = re.compile(
+    r"[0-9a-z]+"
+    r"|[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]"
+)
+
+#: Keys that hold a citation's text across the shipped engines.
+SOURCE_TEXT_KEYS = ("content", "text", "snippet", "excerpt", "summary")
+
+
+def normalize_text(value: str) -> str:
+    """Lower-case ``value`` and collapse whitespace for containment checks."""
+    return _WHITESPACE.sub(" ", (value or "").strip().lower())
+
+
+def tokenize(value: str) -> list[str]:
+    """Split ``value`` into comparison tokens (ASCII words + CJK characters)."""
+    return _TOKEN.findall((value or "").lower())
+
+
+def source_text(source: Any) -> str:
+    """Return the text of one citation record, across the engines' shapes.
+
+    Every pipeline normalizes what it retrieved into ``content``, but a
+    hand-written record or a future engine may use another key; an unrecognized
+    or non-mapping record yields an empty string, which never matches.
+    """
+    if not isinstance(source, Mapping):
+        return ""
+    for key in SOURCE_TEXT_KEYS:
+        value = source.get(key)
+        if isinstance(value, str) and value.strip():
+            return value
+    return ""
+
+
+@dataclass(frozen=True)
+class MatchPolicy:
+    """Thresholds for treating a retrieved chunk as covering a gold passage."""
+
+    min_ratio: float = DEFAULT_MIN_RATIO
+    min_tokens: int = DEFAULT_MIN_TOKENS
+
+    def __post_init__(self) -> None:
+        """Reject thresholds that would make matching either trivial or impossible."""
+        if not 0.0 < float(self.min_ratio) <= 1.0:
+            raise ValueError("min_ratio must be in (0, 1]")
+        if int(self.min_tokens) < 1:
+            raise ValueError("min_tokens must be >= 1")
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    """Which gold passages a chunk covers, plus the strongest coverage ratio."""
+
+    indices: tuple[int, ...] = ()
+    ratio: float = 0.0
+
+    def __bool__(self) -> bool:
+        """True when the chunk covers at least one gold passage."""
+        return bool(self.indices)
+
+
+@dataclass(frozen=True)
+class GoldPassage:
+    """A gold passage prepared once, then matched against every retrieved chunk."""
+
+    index: int
+    text: str
+    normalized: str
+    tokens: Counter[str]
+    token_count: int
+
+    @classmethod
+    def from_text(cls, index: int, text: str) -> GoldPassage:
+        """Tokenize one gold passage for repeated comparison."""
+        token_list = tokenize(text)
+        return cls(
+            index=index,
+            text=text,
+            normalized=normalize_text(text),
+            tokens=Counter(token_list),
+            token_count=len(token_list),
+        )
+
+
+def _overlap(
+    chunk_tokens: Counter[str],
+    chunk_count: int,
+    passage: GoldPassage,
+) -> tuple[float, int]:
+    """Return ``(ratio, shared)`` for one chunk/passage pair.
+
+    The ratio divides by the shorter side's token count, so a truncated citation
+    whose tokens are all inside the passage still scores 1.0.
+    """
+    if chunk_count <= 0 or passage.token_count <= 0:
+        return 0.0, 0
+    shared = 0
+    for token, count in chunk_tokens.items():
+        passage_count = passage.tokens.get(token, 0)
+        if passage_count:
+            shared += min(count, passage_count)
+    shorter = min(chunk_count, passage.token_count)
+    return shared / shorter, shared
+
+
+class RelevanceMatcher:
+    """Decide whether retrieved chunks cover one case's gold passages.
+
+    The matcher is built per case: it prepares each gold passage once and then
+    scores every ranked chunk against them.
+
+    Args:
+        gold: The case's gold excerpts, in dataset order.
+        policy: Matching thresholds; defaults to :class:`MatchPolicy`.
+    """
+
+    def __init__(self, gold: Sequence[str], policy: MatchPolicy | None = None) -> None:
+        self.policy = policy or MatchPolicy()
+        self.gold: tuple[GoldPassage, ...] = tuple(
+            GoldPassage.from_text(index, passage)
+            for index, passage in enumerate(gold)
+            if isinstance(passage, str) and passage.strip()
+        )
+
+    @property
+    def gold_count(self) -> int:
+        """Number of non-blank gold passages this matcher scored against."""
+        return len(self.gold)
+
+    def match(self, text: str) -> MatchResult:
+        """Return the gold passages ``text`` covers and the best coverage ratio."""
+        normalized = normalize_text(text)
+        if not normalized or not self.gold:
+            return MatchResult()
+
+        chunk_tokens = Counter(tokenize(text))
+        chunk_count = sum(chunk_tokens.values())
+
+        indices: list[int] = []
+        best = 0.0
+        for passage in self.gold:
+            ratio = self._ratio(normalized, chunk_tokens, chunk_count, passage)
+            if ratio > 0.0:
+                indices.append(passage.index)
+                best = max(best, ratio)
+        return MatchResult(tuple(indices), round(best, 4))
+
+    def _ratio(
+        self,
+        normalized: str,
+        chunk_tokens: Counter[str],
+        chunk_count: int,
+        passage: GoldPassage,
+    ) -> float:
+        """Coverage ratio for one passage (0.0 when it is not a match)."""
+        if not passage.normalized:
+            return 0.0
+        if passage.normalized in normalized:
+            return 1.0
+        # A truncated citation is shorter than the passage it quotes, so an
+        # exact reverse containment is only meaningful once it carries enough
+        # tokens to be more than an incidental substring.
+        if chunk_count >= max(1, int(self.policy.min_tokens)) and normalized in passage.normalized:
+            return 1.0
+
+        ratio, shared = _overlap(chunk_tokens, chunk_count, passage)
+        if ratio < float(self.policy.min_ratio) or shared < int(self.policy.min_tokens):
+            return 0.0
+        return ratio
+
+
+__all__ = [
+    "DEFAULT_MIN_RATIO",
+    "DEFAULT_MIN_TOKENS",
+    "GoldPassage",
+    "MatchPolicy",
+    "MatchResult",
+    "RelevanceMatcher",
+    "SOURCE_TEXT_KEYS",
+    "normalize_text",
+    "source_text",
+    "tokenize",
+]

deeptutor/services/rag/eval/metrics.py

@@ -0,0 +1,213 @@
+"""Rank-aware retrieval metrics for knowledge-base evaluation.
+
+Every metric is computed at a rank cutoff ``k`` from the *hits* a case produced:
+``hits[i]`` is the set of gold-passage indices the chunk at rank ``i + 1`` covers
+(see :mod:`deeptutor.services.rag.eval.matching`). Working from a set per rank —
+rather than a single relevant/irrelevant flag — is what lets Recall count each
+gold passage once even when two chunks quote it, and lets MAP credit the rank at
+which each *new* passage first appears.
+
+No metric here calls a model: they are pure functions over rankings, so a score
+is reproducible from a stored report and cannot drift with a judge's sampling.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Sequence
+from dataclasses import dataclass
+import math
+
+#: A per-rank record of which gold passages that rank covered.
+Hits = Sequence[Collection[int]]
+
+#: QueryMetrics attributes in the order of :func:`metric_keys`.
+_METRIC_ATTRS = ("recall", "precision", "ndcg", "mrr", "hit", "average_precision")
+
+
+def metric_keys(k: int) -> tuple[str, ...]:
+    """Report keys for a cutoff ``k``, in display order."""
+    _require_k(k)
+    return (f"recall@{k}", f"precision@{k}", f"ndcg@{k}", "mrr", f"hit@{k}", f"map@{k}")
+
+
+def _require_k(k: int) -> int:
+    """Validate a rank cutoff, returning it as an int."""
+    value = int(k)
+    if value < 1:
+        raise ValueError("k must be >= 1")
+    return value
+
+
+def _window(hits: Hits, k: int | None) -> Sequence[Collection[int]]:
+    """Truncate a ranking to the cutoff (``None`` keeps every retrieved rank)."""
+    if k is None:
+        return hits
+    return hits[: _require_k(k)]
+
+
+def hit_flags(hits: Hits, k: int | None = None) -> list[bool]:
+    """Whether each rank (up to ``k``) covered at least one gold passage."""
+    return [bool(matched) for matched in _window(hits, k)]
+
+
+def recall_at_k(hits: Hits, gold_count: int, k: int) -> float:
+    """Share of the case's gold passages retrieved within the top ``k``."""
+    cutoff = _require_k(k)
+    if gold_count <= 0:
+        return 0.0
+    found: set[int] = set()
+    for matched in hits[:cutoff]:
+        found.update(matched)
+    return min(1.0, len(found) / gold_count)
+
+
+def precision_at_k(hits: Hits, k: int) -> float:
+    """Share of the retrieved chunks within ``k`` that covered a gold passage.
+
+    The denominator is the number of chunks actually retrieved, so a short
+    knowledge base is not punished for returning fewer than ``k`` chunks.
+    """
+    cutoff = _require_k(k)
+    window = hits[:cutoff]
+    if not window:
+        return 0.0
+    return sum(1 for matched in window if matched) / len(window)
+
+
+def hit_at_k(hits: Hits, k: int) -> float:
+    """``1.0`` when any gold passage reached the top ``k`` — the citation hit rate."""
+    return 1.0 if any(_window(hits, k)) else 0.0
+
+
+def reciprocal_rank(hits: Hits, k: int | None = None) -> float:
+    """Reciprocal rank of the first chunk that covered a gold passage (MRR@k)."""
+    for rank, matched in enumerate(_window(hits, k), start=1):
+        if matched:
+            return 1.0 / rank
+    return 0.0
+
+
+def average_precision(hits: Hits, gold_count: int, k: int | None = None) -> float:
+    """Mean of the precision measured each time a *new* gold passage appears."""
+    if gold_count <= 0:
+        return 0.0
+    seen: set[int] = set()
+    total = 0.0
+    for rank, matched in enumerate(_window(hits, k), start=1):
+        fresh = {index for index in matched if index not in seen}
+        if not fresh:
+            continue
+        seen.update(fresh)
+        total += len(seen) / rank
+    return min(1.0, total / gold_count)
+
+
+def _dcg(gains: Sequence[float]) -> float:
+    """Discounted cumulative gain with the standard ``log2(rank + 1)`` discount."""
+    return sum(gain / math.log2(rank + 1) for rank, gain in enumerate(gains, start=1))
+
+
+def ndcg_at_k(hits: Hits, gold_count: int, k: int) -> float:
+    """Binary nDCG@k — whether the gold passages are ranked near the top.
+
+    The ideal ranking holds the case's gold passages (capped at ``k``) ahead of
+    everything else, so a case whose gold is all retrieved but buried below
+    irrelevant chunks scores below 1. Each rank contributes at most one gain,
+    and only when it introduces a gold passage not covered by an earlier rank.
+    Repeating the same passage cannot substitute for missing gold passages.
+    """
+    cutoff = _require_k(k)
+    if gold_count <= 0:
+        return 0.0
+    seen: set[int] = set()
+    gains: list[float] = []
+    for matched in hits[:cutoff]:
+        gains.append(1.0 if set(matched) - seen else 0.0)
+        seen.update(matched)
+    ideal_dcg = _dcg([1.0] * min(gold_count, cutoff))
+    if ideal_dcg == 0:
+        return 0.0
+    return min(1.0, _dcg(gains) / ideal_dcg)
+
+
+@dataclass(frozen=True)
+class QueryMetrics:
+    """One case's retrieval scores, plus the counts a report should show."""
+
+    query_id: str
+    k: int
+    retrieved: int
+    relevant_retrieved: int
+    gold_count: int
+    recall: float
+    precision: float
+    ndcg: float
+    mrr: float
+    hit: float
+    average_precision: float
+
+    def values(self) -> dict[str, float]:
+        """Metric values keyed the way reports display them."""
+        keys = metric_keys(self.k)
+        return dict(zip(keys, (getattr(self, name) for name in _METRIC_ATTRS)))
+
+    def to_dict(self) -> dict[str, float | int | str]:
+        """Counts plus rounded metrics, ready for JSON output."""
+        return {
+            "query_id": self.query_id,
+            "k": self.k,
+            "retrieved": self.retrieved,
+            "relevant_retrieved": self.relevant_retrieved,
+            "gold_count": self.gold_count,
+            **{key: round(value, 4) for key, value in self.values().items()},
+        }
+
+
+def evaluate_hits(query_id: str, hits: Hits, gold_count: int, k: int) -> QueryMetrics:
+    """Score one case's ranking at cutoff ``k``."""
+    cutoff = _require_k(k)
+    window = hits[:cutoff]
+    return QueryMetrics(
+        query_id=query_id,
+        k=cutoff,
+        retrieved=len(hits),
+        relevant_retrieved=sum(1 for matched in window if matched),
+        gold_count=max(0, int(gold_count)),
+        recall=recall_at_k(hits, gold_count, cutoff),
+        precision=precision_at_k(hits, cutoff),
+        ndcg=ndcg_at_k(hits, gold_count, cutoff),
+        mrr=reciprocal_rank(hits, cutoff),
+        hit=hit_at_k(hits, cutoff),
+        average_precision=average_precision(hits, gold_count, cutoff),
+    )
+
+
+def aggregate_query_metrics(items: Sequence[QueryMetrics], k: int) -> dict[str, float | int]:
+    """Macro-average the metrics of ``items`` at cutoff ``k``.
+
+    An empty list yields zeros rather than an error, so a report always has a
+    metrics block to print.
+    """
+    keys = metric_keys(k)
+    averages = {key: 0.0 for key in keys}
+    if items:
+        for key, name in zip(keys, _METRIC_ATTRS):
+            total = sum(getattr(item, name) for item in items)
+            averages[key] = round(total / len(items), 4)
+    return {**averages, "queries": len(items)}
+
+
+__all__ = [
+    "Hits",
+    "QueryMetrics",
+    "aggregate_query_metrics",
+    "average_precision",
+    "evaluate_hits",
+    "hit_at_k",
+    "hit_flags",
+    "metric_keys",
+    "ndcg_at_k",
+    "precision_at_k",
+    "recall_at_k",
+    "reciprocal_rank",
+]

deeptutor/services/rag/eval/report.py

@@ -0,0 +1,111 @@
+"""Report shape for a retrieval-evaluation run.
+
+The report is plain data plus two renderings: :meth:`EvalReport.to_json` for a
+stored artifact (diffable between engine or settings changes) and
+:meth:`EvalReport.summary_lines` for a one-block CLI summary. Rich table building
+stays in the CLI, so this module carries no rendering dependency.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from typing import Any
+
+from .matching import MatchPolicy
+from .metrics import aggregate_query_metrics, metric_keys
+from .runner import SOURCE_MODE_CONTEXT, QueryEvaluation
+
+#: Decimals used when a metric is printed.
+METRIC_DECIMALS = 3
+
+
+def format_metric(value: float | int) -> str:
+    """Format one metric for a table cell or a summary line."""
+    return f"{float(value):.{METRIC_DECIMALS}f}"
+
+
+@dataclass(frozen=True)
+class EvalReport:
+    """Everything one evaluation run produced."""
+
+    dataset_name: str
+    kb_name: str
+    provider: str
+    k: int
+    mode: str | None
+    policy: MatchPolicy
+    evaluations: tuple[QueryEvaluation, ...]
+    duration_seconds: float = 0.0
+
+    @property
+    def scored(self) -> tuple[QueryEvaluation, ...]:
+        """Cases that produced a ranking and were scored."""
+        return tuple(item for item in self.evaluations if not item.failed)
+
+    @property
+    def failed(self) -> tuple[QueryEvaluation, ...]:
+        """Cases whose search failed, excluded from the aggregate."""
+        return tuple(item for item in self.evaluations if item.failed)
+
+    def metric_columns(self) -> tuple[str, ...]:
+        """Column headers for the per-case metrics table."""
+        return metric_keys(self.k)
+
+    def format_metrics(self, evaluation: QueryEvaluation) -> dict[str, str]:
+        """Formatted metric cells for one case, keyed by :meth:`metric_columns`."""
+        values = evaluation.metrics.values()
+        return {key: format_metric(values[key]) for key in self.metric_columns()}
+
+    def aggregate(self) -> dict[str, float | int]:
+        """Macro-averaged metrics over the scored cases, plus the case counts."""
+        summary = aggregate_query_metrics([item.metrics for item in self.scored], self.k)
+        summary["queries_scored"] = len(self.scored)
+        summary["queries_failed"] = len(self.failed)
+        return summary
+
+    def to_dict(self) -> dict[str, Any]:
+        """Full serializable report, including per-case detail."""
+        return {
+            "dataset": self.dataset_name,
+            "kb": self.kb_name,
+            "provider": self.provider,
+            "k": self.k,
+            "mode": self.mode or "",
+            "duration_seconds": round(self.duration_seconds, 2),
+            "match_policy": {
+                "min_ratio": self.policy.min_ratio,
+                "min_tokens": self.policy.min_tokens,
+            },
+            "metrics": self.aggregate(),
+            "queries": [
+                {
+                    "query": item.query,
+                    "error": item.error,
+                    "context_only": item.source_mode == SOURCE_MODE_CONTEXT,
+                    **item.to_dict(),
+                }
+                for item in self.evaluations
+            ],
+        }
+
+    def to_json(self, *, indent: int = 2) -> str:
+        """Serialize the report for ``--save`` or ``--format json``."""
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
+
+    def summary_lines(self) -> list[str]:
+        """The header lines a CLI prints under the per-case table."""
+        summary = self.aggregate()
+        lines = [
+            f"dataset: {self.dataset_name}  kb: {self.kb_name}  provider: {self.provider or 'unknown'}",
+            f"k: {self.k}  mode: {self.mode or 'engine default'}",
+            f"cases: {len(self.evaluations)}  scored: {len(self.scored)}  failed: {len(self.failed)}",
+        ]
+        metrics = "  ".join(f"{key}={format_metric(summary[key])}" for key in self.metric_columns())
+        lines.append(f"metrics (mean over scored cases): {metrics}")
+        if self.failed:
+            lines.append("failed cases carry no score and are excluded from the means")
+        return lines
+
+
+__all__ = ["EvalReport", "format_metric", "METRIC_DECIMALS"]