deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/eval/dataset.py

@@ -0,0 +1,244 @@
+"""QA-set schema for knowledge-base retrieval evaluation.
+
+An evaluation set is a list of cases; each case pairs one natural-language
+query with the source passages an ideal retrieval should return for it. The
+passages ("gold") are verbatim excerpts of the indexed documents, so a case
+stays meaningful across re-indexing, re-embedding, and engine changes — a chunk
+identifier does not survive an index rebuild, but document text does.
+
+Two on-disk shapes are supported:
+
+* ``.jsonl`` / ``.ndjson`` — one case object per line (recommended: a long set
+  stays reviewable in a diff);
+* ``.json`` — either a bare list of cases or ``{"name": ..., "cases": [...]}``.
+
+Case fields:
+
+* ``query`` (required) — the question or phrase handed to retrieval.
+* ``gold`` (required) — one excerpt or a list of excerpts that should be
+  retrieved. A single string is accepted as a one-passage list.
+* ``id`` (optional) — a stable label used in reports; defaults to ``q1``, ``q2``…
+* ``notes`` (optional) — free text carried through to the report.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterator, Sequence
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+
+JSONL_SUFFIXES = frozenset({".jsonl", ".ndjson"})
+
+
+class EvalDatasetError(ValueError):
+    """Raised when an evaluation set cannot be read or is structurally invalid."""
+
+
+class EvalCase(BaseModel):
+    """One query with the gold passages that should be retrieved for it."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    query: str
+    gold: list[str] = Field(min_length=1)
+    id: str = ""
+    notes: str = ""
+
+    @field_validator("query")
+    @classmethod
+    def _require_query(cls, value: str) -> str:
+        """Reject a blank query — there is nothing to retrieve for it."""
+        text = (value or "").strip()
+        if not text:
+            raise ValueError("query must be a non-empty string")
+        return text
+
+    @field_validator("gold", mode="before")
+    @classmethod
+    def _coerce_gold(cls, value: Any) -> Any:
+        """Accept a single excerpt string as a one-passage gold list."""
+        if isinstance(value, str):
+            return [value]
+        return value
+
+    @field_validator("gold")
+    @classmethod
+    def _require_gold(cls, value: list[str]) -> list[str]:
+        """Reject an empty or blank gold list — matching would be meaningless."""
+        passages = [(item or "").strip() for item in value]
+        if not passages or any(not passage for passage in passages):
+            raise ValueError("gold must hold at least one non-empty passage")
+        return passages
+
+    @field_validator("id", "notes", mode="before")
+    @classmethod
+    def _coerce_text(cls, value: Any) -> str:
+        """Treat a missing/null label or note as an empty string."""
+        return "" if value is None else str(value).strip()
+
+
+class EvalDataset(BaseModel):
+    """A named collection of evaluation cases."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    name: str = "eval-set"
+    cases: list[EvalCase] = Field(min_length=1)
+    source: str = ""
+
+    def __len__(self) -> int:
+        """Number of cases in the set."""
+        return len(self.cases)
+
+    def iter_cases(self) -> Iterator[EvalCase]:
+        """Iterate the cases in file order."""
+        return iter(self.cases)
+
+    def limited(self, limit: int | None) -> EvalDataset:
+        """Return a copy holding at most ``limit`` cases (``None`` keeps all)."""
+        if limit is None or limit >= len(self.cases):
+            return self
+        return self.model_copy(update={"cases": self.cases[: max(0, limit)]})
+
+    @classmethod
+    def from_cases(cls, cases: Sequence[EvalCase | dict], *, name: str = "eval-set") -> EvalDataset:
+        """Build a dataset programmatically (used by tests and scripts)."""
+        return cls(name=name, cases=[_coerce_case(case, index) for index, case in enumerate(cases)])
+
+
+def _coerce_case(case: EvalCase | dict, index: int) -> EvalCase:
+    """Validate one in-memory case, filling in a default id."""
+    if isinstance(case, EvalCase):
+        return case if case.id else case.model_copy(update={"id": f"q{index + 1}"})
+    validated = EvalCase.model_validate(case)
+    return validated if validated.id else validated.model_copy(update={"id": f"q{index + 1}"})
+
+
+def _first_error(exc: ValidationError) -> str:
+    """Flatten a pydantic error into one readable line."""
+    error = exc.errors()[0]
+    location = ".".join(str(part) for part in error.get("loc", ())) or "case"
+    return f"{location}: {error.get('msg', 'invalid value')}"
+
+
+def _case_context(raw: Any) -> str:
+    """A short excerpt of the offending case, to make errors actionable."""
+    if isinstance(raw, dict):
+        query = raw.get("query")
+        if isinstance(query, str) and query.strip():
+            return f" (query: {query.strip()[:48]!r})"
+    return ""
+
+
+def _build_case(raw: Any, index: int, path: Path) -> EvalCase:
+    """Validate one raw case object, raising with file/line context."""
+    if not isinstance(raw, dict):
+        raise EvalDatasetError(
+            f"{path}: case #{index + 1} must be a JSON object, got {type(raw).__name__}."
+        )
+    try:
+        case = EvalCase.model_validate(raw)
+    except ValidationError as exc:
+        raise EvalDatasetError(
+            f"{path}: case #{index + 1}{_case_context(raw)} is invalid — {_first_error(exc)}"
+        ) from exc
+    return case if case.id else case.model_copy(update={"id": f"q{index + 1}"})
+
+
+def _parse_jsonl(text: str, path: Path) -> list[Any]:
+    """Parse one case object per non-blank line, reporting the failing line."""
+    cases: list[Any] = []
+    for line_number, line in enumerate(text.splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            cases.append(json.loads(line))
+        except json.JSONDecodeError as exc:
+            raise EvalDatasetError(
+                f"{path}: line {line_number} is not valid JSON — {exc.msg}"
+            ) from exc
+    return cases
+
+
+def _parse_json(text: str, path: Path) -> tuple[list[Any], str]:
+    """Parse either a bare case list or an object with ``name`` and ``cases``."""
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise EvalDatasetError(f"{path}: is not valid JSON — {exc.msg}") from exc
+    if isinstance(payload, list):
+        return payload, ""
+    if isinstance(payload, dict):
+        cases = payload.get("cases")
+        if not isinstance(cases, list):
+            raise EvalDatasetError(f"{path}: object form must hold a 'cases' list.")
+        name = payload.get("name")
+        return cases, str(name).strip() if isinstance(name, str) else ""
+    raise EvalDatasetError(f"{path}: must hold a list of cases or an object with 'cases'.")
+
+
+def _reject_duplicate_ids(cases: Sequence[EvalCase], path: Path) -> None:
+    """Fail on repeated case ids so report rows cannot be ambiguous."""
+    seen: set[str] = set()
+    for case in cases:
+        if case.id in seen:
+            raise EvalDatasetError(f"{path}: duplicate case id {case.id!r}.")
+        seen.add(case.id)
+
+
+def load_dataset(path: str | Path, *, name: str = "") -> EvalDataset:
+    """Read an evaluation set from ``path``.
+
+    Args:
+        path: A ``.jsonl``, ``.ndjson`` or ``.json`` file.
+        name: Overrides the set name; defaults to the file's ``name`` field,
+            then to the file stem.
+
+    Returns:
+        The parsed, validated dataset.
+
+    Raises:
+        EvalDatasetError: The file is missing, unreadable, of an unsupported
+            format, empty, or holds an invalid or duplicated case.
+    """
+    target = Path(path).expanduser()
+    if not target.exists():
+        raise EvalDatasetError(f"Evaluation set not found: {target}")
+    if target.is_dir():
+        raise EvalDatasetError(f"Evaluation set must be a file, not a directory: {target}")
+
+    suffix = target.suffix.lower()
+    if suffix not in JSONL_SUFFIXES and suffix != ".json":
+        raise EvalDatasetError(
+            f"Unsupported evaluation set format {target.suffix!r}; use .jsonl or .json."
+        )
+
+    try:
+        text = target.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise EvalDatasetError(f"Could not read {target}: {exc}") from exc
+    except UnicodeDecodeError as exc:
+        raise EvalDatasetError(f"{target} is not UTF-8 text: {exc}") from exc
+
+    if suffix in JSONL_SUFFIXES:
+        raw_cases, set_name = _parse_jsonl(text, target), ""
+    else:
+        raw_cases, set_name = _parse_json(text, target)
+
+    if not raw_cases:
+        raise EvalDatasetError(f"Evaluation set {target} holds no cases.")
+
+    cases = [_build_case(raw, index, target) for index, raw in enumerate(raw_cases)]
+    _reject_duplicate_ids(cases, target)
+    return EvalDataset(name=name or set_name or target.stem, cases=cases, source=str(target))
+
+
+__all__ = [
+    "EvalCase",
+    "EvalDataset",
+    "EvalDatasetError",
+    "JSONL_SUFFIXES",
+    "load_dataset",
+]

deeptutor/services/rag/eval/diff.py

@@ -0,0 +1,312 @@
+"""Compare two retrieval-evaluation reports to quantify a settings change.
+
+A single ``kb eval`` run reports absolute metrics; it cannot tell you whether a
+change to the retrieval stack (an embedding model, a reranker, chunk size, or the
+query-rewrite layer) actually helped. This module answers that by diffing two
+stored reports: a *baseline* recorded before the change and a *candidate* recorded
+after. It aligns the two by query id and reports the per-metric delta plus the
+per-case movement, so a reordered or truncated set still compares like-for-like.
+
+Nothing here calls a model: every number is recomputed from the two reports, so
+the diff is reproducible and cannot drift.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from .metrics import metric_keys
+from .report import format_metric
+
+#: A delta above this magnitude counts as a move (symmetric for regressions).
+_IMPROVE_EPSILON = 1e-9
+
+
+@dataclass(frozen=True)
+class _Normalized:
+    """A report reduced to just what a diff needs: ``k``, metric keys, metrics, cases.
+
+    Both a live :class:`~deeptutor.services.rag.eval.report.EvalReport` object and a
+    serialized ``to_dict`` (e.g. loaded from a ``--save`` JSON file) normalize to the
+    same shape, so a diff works without reconstructing report objects.
+    """
+
+    k: int
+    keys: tuple[str, ...]
+    metrics: dict[str, float]
+    cases: dict[str, dict[str, Any]]
+
+
+def _normalize(report: Any) -> _Normalized:
+    """Reduce a report object or its ``to_dict`` to the fields a diff compares.
+
+    Args:
+        report: A report object or its serialized ``to_dict`` mapping.
+
+    Returns:
+        A :class:`_Normalized` holding the macro metrics and per-case scores.
+
+    Raises:
+        TypeError: If ``report`` is neither a report object nor a mapping.
+    """
+    if isinstance(report, dict):
+        k = int(report.get("k") or 0)
+        aggregate: dict[str, Any] = report.get("metrics") or {}
+        raw_cases: Sequence[Any] = report.get("queries") or []
+        object_form = False
+    else:
+        k = int(getattr(report, "k", 0) or 0)
+        aggregate = dict(report.aggregate())
+        raw_cases = list(getattr(report, "evaluations", ()))
+        object_form = True
+
+    keys = metric_keys(k) if k >= 1 else ()
+    metrics = {key: float(aggregate.get(key, 0.0) or 0.0) for key in keys}
+
+    cases: dict[str, dict[str, Any]] = {}
+    for case in raw_cases:
+        if object_form:
+            case_id = str(case.case_id)
+            query = str(case.query)
+            failed = case.failed
+            values = case.metrics.values()
+        else:
+            case_id = str(case.get("case_id") or case.get("query_id") or case.get("query"))
+            query = str(case.get("query") or case_id)
+            failed = bool(case.get("error"))
+            values = case
+        metrics_by_key = {key: float(values.get(key, 0.0) or 0.0) for key in keys}
+        cases[case_id] = {"query": query, "failed": failed, "metrics": metrics_by_key}
+
+    return _Normalized(k=k, keys=keys, metrics=metrics, cases=cases)
+
+
+@dataclass(frozen=True)
+class MetricDelta:
+    """One metric's baseline value, candidate value, and signed change."""
+
+    key: str
+    baseline: float
+    candidate: float
+
+    @property
+    def delta(self) -> float:
+        """``candidate - baseline``; positive means the change improved this metric."""
+        return self.candidate - self.baseline
+
+    @property
+    def improved(self) -> bool:
+        """Whether the metric went up by more than a rounding epsilon."""
+        return self.delta > _IMPROVE_EPSILON
+
+    @property
+    def regressed(self) -> bool:
+        """Whether the metric went down by more than a rounding epsilon."""
+        return self.delta < -_IMPROVE_EPSILON
+
+    def to_dict(self) -> dict[str, Any]:
+        """A JSON-serializable row for this metric's delta."""
+        return {
+            "metric": self.key,
+            "baseline": round(self.baseline, 6),
+            "candidate": round(self.candidate, 6),
+            "delta": round(self.delta, 6),
+            "direction": "improved" if self.improved else "regressed" if self.regressed else "flat",
+        }
+
+
+@dataclass(frozen=True)
+class CaseDelta:
+    """One query's metric deltas, comparing only where both runs scored it."""
+
+    query: str
+    baseline_failed: bool
+    candidate_failed: bool
+    metrics: dict[str, MetricDelta] = field(default_factory=dict)
+
+    @property
+    def comparable(self) -> bool:
+        """Whether both runs produced a score for this query."""
+        return not (self.baseline_failed or self.candidate_failed)
+
+    @property
+    def worst_key(self) -> str | None:
+        """The metric with the most negative delta, if the case has metrics."""
+        if not self.metrics:
+            return None
+        return min(self.metrics, key=lambda key: self.metrics[key].delta)
+
+    def to_dict(self) -> dict[str, Any]:
+        """A JSON-serializable row for this case's per-metric deltas."""
+        return {
+            "query": self.query,
+            "comparable": self.comparable,
+            "baseline_failed": self.baseline_failed,
+            "candidate_failed": self.candidate_failed,
+            "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},
+        }
+
+
+@dataclass(frozen=True)
+class EvalDiff:
+    """The per-metric and per-case movement between a baseline and a candidate run.
+
+    Macro deltas cover the scored cases of each run. Per-case deltas are only
+    computed where *both* runs scored the query; a case one run failed (a transient
+    engine error) is reported as added/removed rather than pretending a zero-vs-score
+    delta is meaningful.
+    """
+
+    baseline_label: str
+    candidate_label: str
+    k: int
+    metrics: dict[str, MetricDelta] = field(default_factory=dict)
+    cases: dict[str, CaseDelta] = field(default_factory=dict)
+    added_cases: tuple[str, ...] = ()
+    removed_cases: tuple[str, ...] = ()
+
+    @property
+    def comparable_cases(self) -> tuple[CaseDelta, ...]:
+        """Cases both runs scored, the ones whose per-case movement is meaningful."""
+        return tuple(case for case in self.cases.values() if case.comparable)
+
+    @property
+    def improved_cases(self) -> tuple[CaseDelta, ...]:
+        """Comparable cases where some metric rose and none fell."""
+        return tuple(
+            case
+            for case in self.comparable_cases
+            if any(delta.improved for delta in case.metrics.values())
+            and not any(delta.regressed for delta in case.metrics.values())
+        )
+
+    @property
+    def regressed_cases(self) -> tuple[CaseDelta, ...]:
+        """Comparable cases where any metric fell."""
+        return tuple(
+            case
+            for case in self.comparable_cases
+            if any(delta.regressed for delta in case.metrics.values())
+        )
+
+    @property
+    def has_regression(self) -> bool:
+        """Whether any macro metric or comparable case regressed."""
+        if any(delta.regressed for delta in self.metrics.values()):
+            return True
+        return bool(self.regressed_cases)
+
+    def to_dict(self) -> dict[str, Any]:
+        """A JSON-serializable form of the whole diff."""
+        return {
+            "baseline": self.baseline_label,
+            "candidate": self.candidate_label,
+            "k": self.k,
+            "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},
+            "cases": {case_id: case.to_dict() for case_id, case in self.cases.items()},
+            "added_cases": list(self.added_cases),
+            "removed_cases": list(self.removed_cases),
+        }
+
+    def summary_lines(self) -> list[str]:
+        """Plain-text lines suitable for a CLI: macro table then case movement."""
+        lines = [
+            f"Baseline: {self.baseline_label}",
+            f"Candidate: {self.candidate_label}",
+            f"k={self.k}",
+            "",
+            f"{'metric':<12} {'baseline':>10} {'candidate':>10} {'delta':>10}",
+        ]
+        for key, delta in self.metrics.items():
+            lines.append(
+                f"{key:<12} "
+                f"{format_metric(delta.baseline):>10} "
+                f"{format_metric(delta.candidate):>10} "
+                f"{format_metric(delta.delta):>10}  {delta.to_dict()['direction']}"
+            )
+        lines.append("")
+        lines.append(
+            f"cases: {len(self.comparable_cases)} comparable, "
+            f"{len(self.improved_cases)} improved, {len(self.regressed_cases)} regressed"
+        )
+        if self.added_cases:
+            lines.append(f"added (candidate only): {', '.join(self.added_cases)}")
+        if self.removed_cases:
+            lines.append(f"removed (baseline only): {', '.join(self.removed_cases)}")
+        return lines
+
+
+def diff_reports(
+    baseline: Any,
+    candidate: Any,
+    *,
+    baseline_label: str = "baseline",
+    candidate_label: str = "candidate",
+) -> EvalDiff:
+    """Diff two evaluation reports (objects or ``to_dict`` mappings).
+
+    Args:
+        baseline: The pre-change report (object or serialized mapping).
+        candidate: The post-change report (object or serialized mapping).
+        baseline_label: Human label for the baseline in the summary.
+        candidate_label: Human label for the candidate in the summary.
+
+    Returns:
+        An :class:`EvalDiff` with per-metric and per-case movement. Cases are
+        aligned by query id; a case scored by only one run is reported as
+        added/removed rather than as a spurious delta.
+
+    Raises:
+        ValueError: If the two reports were evaluated at different ``k`` and so
+            cannot be compared like-for-like.
+    """
+    base = _normalize(baseline)
+    cand = _normalize(candidate)
+    if base.k != cand.k:
+        raise ValueError(
+            f"reports use different k (baseline k={base.k}, candidate k={cand.k}); "
+            "re-evaluate both at the same k to compare"
+        )
+
+    metrics = {
+        key: MetricDelta(
+            key=key, baseline=base.metrics.get(key, 0.0), candidate=cand.metrics.get(key, 0.0)
+        )
+        for key in base.keys
+    }
+
+    base_ids = set(base.cases)
+    cand_ids = set(cand.cases)
+    shared = sorted(base_ids & cand_ids)
+
+    cases: dict[str, CaseDelta] = {}
+    for case_id in shared:
+        base_case = base.cases[case_id]
+        cand_case = cand.cases[case_id]
+        comparable = not (base_case["failed"] or cand_case["failed"])
+        per_metric = {
+            key: MetricDelta(
+                key=key,
+                baseline=base_case["metrics"].get(key, 0.0),
+                candidate=cand_case["metrics"].get(key, 0.0),
+            )
+            for key in base.keys
+        }
+        cases[case_id] = CaseDelta(
+            query=cand_case["query"] or base_case["query"],
+            baseline_failed=base_case["failed"],
+            candidate_failed=cand_case["failed"],
+            metrics=per_metric if comparable else {},
+        )
+
+    return EvalDiff(
+        baseline_label=baseline_label,
+        candidate_label=candidate_label,
+        k=base.k,
+        metrics=metrics,
+        cases=cases,
+        added_cases=tuple(sorted(cand_ids - base_ids)),
+        removed_cases=tuple(sorted(base_ids - cand_ids)),
+    )