retrieval-eval-gate 0.1.0__py3-none-any.whl

rag_eval_gate/__init__.py

@@ -0,0 +1,39 @@
+"""Retrieval evaluation that reports what it can and cannot resolve.
+
+A score is an estimate from a finite query sample, so nothing here reports a bare number.
+Metrics arrive with intervals, comparisons with p-values corrected for having asked several
+questions at once, and every gold set with a statement of the differences it is too small
+to detect.
+
+    from rag_eval_gate import HttpBackend, load_gold, evaluate, add_intervals
+
+    queries = load_gold("gold.jsonl")
+    backend = HttpBackend("http://localhost:8080/api/search?q={query}&mode={mode}")
+    result = evaluate(backend, queries, "hybrid")
+    add_intervals({"hybrid": result}, iters=10_000)
+    print(result.means["mrr@10"], result.intervals["mrr@10"])
+
+The statistical layer is usable on its own — `stats` for intervals and paired tests,
+`sequential` for anytime-valid gates, `conformal` for distribution-free guarantees, and
+`ppi` for a valid interval on a quantity only a biased LLM judge has scored at scale.
+"""
+from .audit import (ComparisonAudit, MetricAudit, audit_comparison, audit_metric,
+                    differing_query_bounds)
+from .backends import (Backend, BackendError, CommandBackend, Hit, HttpBackend,
+                       RunFileBackend, build_backend)
+from .goldset import GoldQuery, GoldSetError, load_gold, shape
+from .harness import (EvalRun, ModeResult, add_intervals, compare_modes, design_of,
+                      evaluate, evaluate_sequential)
+from .metrics import METRICS, QueryScore, aggregate, score_query, values_of
+from .ppi import PPIEstimate, classical_interval, ppi_mean
+from .version import __version__
+
+__all__ = [
+    "MetricAudit", "ComparisonAudit", "audit_metric", "audit_comparison",
+    "differing_query_bounds",
+    "Backend", "BackendError", "CommandBackend", "Hit", "HttpBackend", "RunFileBackend",
+    "build_backend", "GoldQuery", "GoldSetError", "load_gold", "shape",
+    "EvalRun", "ModeResult", "add_intervals", "compare_modes", "design_of", "evaluate",
+    "evaluate_sequential", "METRICS", "QueryScore", "aggregate", "score_query",
+    "values_of", "PPIEstimate", "ppi_mean", "classical_interval", "__version__",
+]

rag_eval_gate/audit.py

@@ -0,0 +1,257 @@
+"""Audit a published number without the data behind it.
+
+Every retrieval paper, blog post and README has a table like
+
+    | mode   | Recall@5 | MRR@10 |
+    | bm25   | 0.70     | 0.67   |
+    | hybrid | 1.00     | 0.95   |
+
+and almost none of them say how many queries it took or what the numbers can support. That
+is usually answerable anyway: a mean over `n` items, on a metric bounded in [0, 1], already
+pins down more than most authors realise.
+
+Three questions get answered from the table alone:
+
+1. **How wide is that number really?** With one relevant document per query, Recall@k is a
+   count of successes and the exact binomial interval applies — no other information needed.
+   For a graded metric the variance is unknown but bounded: a [0, 1] variable with mean `m`
+   has variance at most `m(1-m)`, so there is a widest-possible interval and it is worth
+   knowing before quoting three decimal places.
+
+2. **Could the reported lift have been significant?** A paired randomization test on `k`
+   differing queries cannot return a two-sided p below `2/2^k`, and the two reported means
+   bound `k` without needing the per-query data. When even the most favourable overlap
+   leaves that floor above 0.05, the comparison could not have reached significance — which
+   is a fact about the table, not a criticism of the system.
+
+3. **What would it take?** The queries needed to resolve an improvement of a given size.
+
+Nothing here is a substitute for the per-query data; it is what can be said when the
+per-query data is not on offer, which is the usual case when reading someone else's results.
+"""
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+
+from . import stats
+
+__all__ = ["MetricAudit", "ComparisonAudit", "audit_metric", "audit_comparison",
+           "differing_query_bounds", "render"]
+
+ALPHA = 0.05
+POWER = 0.80
+# A reported mean that is not a clean multiple of 1/n cannot be a count of successes.
+BINARY_TOLERANCE = 1e-6
+
+
+@dataclass
+class MetricAudit:
+    metric: str
+    value: float
+    n: int
+    binary: bool
+    lo: float
+    hi: float
+    method: str
+    successes: int | None = None
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def width(self) -> float:
+        return self.hi - self.lo
+
+    def as_dict(self) -> dict:
+        return {"metric": self.metric, "value": self.value, "n": self.n,
+                "binary": self.binary, "lo": self.lo, "hi": self.hi,
+                "width": self.width, "method": self.method,
+                "successes": self.successes, "notes": self.notes}
+
+
+@dataclass
+class ComparisonAudit:
+    metric: str
+    value: float
+    baseline: float
+    n: int
+    binary: bool
+    delta: float
+    k_min: int
+    k_max: int
+    floor_p_best: float          # the smallest p attainable, under the kindest overlap
+    floor_p_worst: float         # ... and under the least kind
+    resolvable: bool             # can *any* overlap reach p < 0.05?
+    determined: bool             # do the two means pin k down exactly?
+    queries_for_delta: int | None
+    notes: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict:
+        return {"metric": self.metric, "value": self.value, "baseline": self.baseline,
+                "n": self.n, "binary": self.binary, "delta": self.delta,
+                "k_min": self.k_min, "k_max": self.k_max,
+                "floor_p_best": self.floor_p_best, "floor_p_worst": self.floor_p_worst,
+                "resolvable": self.resolvable, "determined": self.determined,
+                "queries_for_delta": self.queries_for_delta, "notes": self.notes}
+
+
+def _looks_binary(value: float, n: int) -> bool:
+    """True when the mean is a clean count of successes over n."""
+    return abs(value * n - round(value * n)) < BINARY_TOLERANCE * max(1, n)
+
+
+def audit_metric(metric: str, value: float, n: int, binary: bool | None = None) -> MetricAudit:
+    """The interval a single reported mean actually supports.
+
+    `binary=None` infers it: Recall@k with one relevant document per query is a per-query
+    0/1 outcome, and its mean is then a multiple of 1/n. The inference is reported rather
+    than assumed silently, because it changes which interval applies.
+    """
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"{metric} must lie in [0, 1], got {value}")
+    if n < 2:
+        raise ValueError(f"need at least 2 queries, got {n}")
+
+    notes: list[str] = []
+    if binary is None:
+        binary = metric.lower().startswith("recall") and _looks_binary(value, n)
+        if binary:
+            notes.append("treated as a per-query 0/1 outcome (Recall@k with one relevant "
+                         "document per query); pass --graded if that is wrong")
+
+    if binary:
+        if not _looks_binary(value, n):
+            raise ValueError(f"{value} is not a multiple of 1/{n}, so it cannot be a "
+                             f"success count over {n} queries")
+        successes = round(value * n)
+        lo, hi = stats.clopper_pearson(successes, n, ALPHA)
+        return MetricAudit(metric, value, n, True, lo, hi, "exact binomial",
+                           successes, notes)
+
+    # Graded metric: the per-query spread is unknown, but a [0, 1] variable with mean m has
+    # variance at most m(1-m) (attained by a Bernoulli). That is the widest the interval can
+    # be — the conservative thing to quote when the per-query scores are not on offer.
+    half = stats.NormalDist().inv_cdf(1 - ALPHA / 2) * math.sqrt(value * (1 - value) / n)
+    notes.append("per-query scores unknown; this is the widest the interval can be, "
+                 "attained when every query scores 0 or 1")
+    return MetricAudit(metric, value, n, False, max(0.0, value - half),
+                       min(1.0, value + half), "maximum-variance bound", None, notes)
+
+
+def differing_query_bounds(value: float, baseline: float, n: int,
+                           binary: bool) -> tuple[int, int, bool]:
+    """How many queries must differ between two systems, given only their means.
+
+    Returns (minimum, maximum, exactly determined).
+
+    For a 0/1 metric with `a` and `b` successes out of `n`, write `d1` for the queries the
+    first system wins and `d2` for those it loses. Then `d1 - d2 = a - b` and the count that
+    differs is `d1 + d2`, with `d1` ranging over `[max(0, a-b), min(a, n-b)]`. When that
+    range collapses to a point — which is what happens against a perfect score — the two
+    published means determine the number exactly.
+
+    For a graded metric each differing query can move the mean by at most `1/n`, so at least
+    `ceil(|delta| * n)` of them differ; the upper bound is `n`.
+    """
+    delta = value - baseline
+    if binary:
+        a, b = round(value * n), round(baseline * n)
+        low_d1, high_d1 = max(0, a - b), min(a, n - b)
+        k_min = 2 * low_d1 - (a - b)
+        k_max = 2 * high_d1 - (a - b)
+        return k_min, k_max, k_min == k_max
+    return math.ceil(abs(delta) * n), n, False
+
+
+def audit_comparison(metric: str, value: float, baseline: float, n: int,
+                     binary: bool | None = None) -> ComparisonAudit:
+    """What a reported lift between two means can and cannot establish."""
+    if binary is None:
+        binary = (metric.lower().startswith("recall")
+                  and _looks_binary(value, n) and _looks_binary(baseline, n))
+    for name, v in (("value", value), ("baseline", baseline)):
+        if not 0.0 <= v <= 1.0:
+            raise ValueError(f"{name} must lie in [0, 1], got {v}")
+    if n < 2:
+        raise ValueError(f"need at least 2 queries, got {n}")
+
+    k_min, k_max, determined = differing_query_bounds(value, baseline, n, binary)
+    # More differing queries means a smaller attainable p, so k_max gives the best case.
+    floor_best = stats.min_attainable_p(k_max)
+    floor_worst = stats.min_attainable_p(k_min)
+
+    notes: list[str] = []
+    if determined:
+        notes.append(f"the two means determine it exactly: {k_min} of {n} queries differ")
+    elif binary:
+        notes.append(f"between {k_min} and {k_max} of {n} queries differ, depending on "
+                     f"which ones each system got right")
+    else:
+        notes.append(f"at least {k_min} of {n} queries differ (each one can move the mean "
+                     f"by at most 1/{n}); the per-query data would pin it down")
+
+    # Sample size to resolve this difference again. A paired test cares about the spread
+    # of the per-query *differences*, not of the raw scores. For a 0/1 metric that spread is
+    # exact given k: each difference is -1, 0 or +1, so Var(d) = k/n - delta^2. Taking the
+    # largest admissible k gives the conservative answer. For a graded metric the
+    # differences are unknown and the [0, 1] variance bound is the best available.
+    delta = value - baseline
+    if binary:
+        spread = math.sqrt(max(0.0, k_max / n - delta ** 2))
+    else:
+        spread = math.sqrt(max(value * (1 - value), baseline * (1 - baseline)))
+    needed = stats.required_queries(abs(delta), spread, ALPHA, POWER)
+
+    return ComparisonAudit(metric, value, baseline, n, binary, value - baseline,
+                           k_min, k_max, floor_best, floor_worst,
+                           resolvable=floor_best < ALPHA, determined=determined,
+                           queries_for_delta=needed, notes=notes)
+
+
+def render(metric_audit: MetricAudit, comparison: ComparisonAudit | None) -> str:
+    out: list[str] = []
+    add = out.append
+    m = metric_audit
+
+    add(f"{m.metric} = {m.value:.3f} on {m.n} queries")
+    if m.successes is not None:
+        add(f"  that is {m.successes}/{m.n} successes")
+    for note in m.notes:
+        add(f"  note: {note}")
+    add("")
+    add(f"  95% interval   [{m.lo:.3f}, {m.hi:.3f}]   width {m.width:.3f}   ({m.method})")
+    if m.width > 0.15:
+        add(f"  The reported figure is {m.width * 100:.0f} points wide. Quoting it to two "
+            f"decimal places implies a precision {m.n} queries cannot deliver.")
+    add("")
+
+    if comparison is None:
+        return "\n".join(out) + "\n"
+
+    c = comparison
+    add(f"Claimed lift {c.delta:+.3f} over {c.baseline:.3f}")
+    for note in c.notes:
+        add(f"  note: {note}")
+    add("")
+    if c.determined:
+        add(f"  smallest attainable p   {c.floor_p_worst:.4f}")
+    else:
+        add(f"  smallest attainable p   {c.floor_p_best:.4f} (best case) to "
+            f"{c.floor_p_worst:.4f} (worst)")
+    add("")
+    if not c.resolvable:
+        add("  UNRESOLVABLE. Under every possible overlap of the two systems' successes,")
+        add("  a paired test on this many differing queries has a floor above 0.05, so no")
+        add("  outcome of that comparison could have reached significance. This is a fact")
+        add("  about the sample size, not about the systems.")
+    elif c.floor_p_worst >= ALPHA:
+        add("  POSSIBLY UNRESOLVABLE. It depends on which queries each system got right —")
+        add("  information the published table does not contain. Ask for the per-query")
+        add("  scores, or for the paired test.")
+    else:
+        add("  Resolvable in principle. Whether it *was* resolved needs the paired test;")
+        add("  a difference in means is not evidence on its own.")
+    add("")
+    if c.queries_for_delta:
+        add(f"  To resolve {abs(c.delta):+.3f} again at 80% power, assuming the widest")
+        add(f"  per-query spread consistent with these means: {c.queries_for_delta:,} queries.")
+    return "\n".join(out) + "\n"

rag_eval_gate/backends.py

@@ -0,0 +1,253 @@
+"""Where rankings come from — HTTP, a subprocess, or a file already on disk.
+
+The evaluation, the statistics and the gate do not care how a ranking was produced, and
+tying them to one API shape is what keeps most eval harnesses locked to the repository
+they were born in. Three adapters cover essentially everything:
+
+* `HttpBackend` — a URL template plus a small field map, for a service that answers JSON.
+  No assumption about the route, the parameter names, or where the hits sit in the body.
+* `CommandBackend` — runs any command and reads JSON from its stdout. A retrieval system
+  written in Go, Rust, TypeScript or a notebook is one shell line away from being gated.
+* `RunFileBackend` — a TREC run file, the format the IR community already exchanges. Rank
+  a corpus once, evaluate it many times, with no service running at all.
+
+All three return the same thing: `Hit` objects in rank order.
+"""
+from __future__ import annotations
+
+import json
+import shlex
+import subprocess
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from pathlib import Path
+
+__all__ = ["Hit", "Backend", "HttpBackend", "CommandBackend", "RunFileBackend",
+           "BackendError", "build_backend"]
+
+
+class BackendError(RuntimeError):
+    """A ranking could not be obtained — distinct from a query legitimately scoring zero."""
+
+
+@dataclass(frozen=True)
+class Hit:
+    """One retrieved item. `score` is optional; only calibration needs it."""
+
+    doc_id: str
+    score: float | None = None
+
+
+class Backend:
+    """Anything that can rank documents for a query."""
+
+    def search(self, query: str, mode: str) -> list[Hit]:      # pragma: no cover - interface
+        raise NotImplementedError
+
+    def close(self) -> None:
+        pass
+
+
+def _dig(payload, path: str):
+    """Follow a dotted path into nested dicts/lists — `results`, `data.hits`, `hits.hits`."""
+    current = payload
+    for part in path.split("."):
+        if part == "":
+            continue
+        if isinstance(current, list):
+            try:
+                current = current[int(part)]
+                continue
+            except (ValueError, IndexError) as e:
+                raise BackendError(f"cannot index list with '{part}' in path '{path}'") from e
+        if not isinstance(current, dict) or part not in current:
+            available = list(current)[:8] if isinstance(current, dict) else type(current).__name__
+            raise BackendError(f"path '{path}': no '{part}' — found {available}")
+        current = current[part]
+    return current
+
+
+@dataclass
+class HttpBackend(Backend):
+    """A search service that answers JSON.
+
+    `url_template` is formatted with `query` (already percent-encoded) and `mode`, so any
+    route shape works:
+
+        http://localhost:8080/api/search?q={query}&mode={mode}
+        https://api.example.com/v2/retrieve?text={query}&strategy={mode}&k=10
+
+    `results_path` locates the hit list in the response body and `id_field` / `score_field`
+    name the keys inside a hit, so no particular schema is assumed. A hit may also be a
+    bare string, in which case it is the document id.
+    """
+
+    url_template: str
+    results_path: str = "results"
+    id_field: str = "docId"
+    score_field: str | None = "score"
+    timeout: float = 120.0
+    attempts: int = 3
+    headers: dict[str, str] = field(default_factory=dict)
+
+    def search(self, query: str, mode: str) -> list[Hit]:
+        url = self.url_template.format(query=urllib.parse.quote(query), mode=mode)
+        payload = self._get(url)
+        rows = _dig(payload, self.results_path)
+        if not isinstance(rows, list):
+            raise BackendError(f"'{self.results_path}' is {type(rows).__name__}, not a list")
+        return [self._hit(row, url) for row in rows]
+
+    def _hit(self, row, url: str) -> Hit:
+        if isinstance(row, str):
+            return Hit(row)
+        if not isinstance(row, dict):
+            raise BackendError(f"hit is {type(row).__name__}, expected object or string")
+        if self.id_field not in row:
+            raise BackendError(f"hit has no '{self.id_field}' — keys: {list(row)[:8]} ({url})")
+        score = row.get(self.score_field) if self.score_field else None
+        return Hit(str(row[self.id_field]), float(score) if score is not None else None)
+
+    def _get(self, url: str):
+        last: Exception | None = None
+        for attempt in range(self.attempts):
+            try:
+                request = urllib.request.Request(
+                    url, headers={"User-Agent": "rag-eval-gate", **self.headers})
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    return json.load(response)
+            except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
+                last = e
+                if attempt + 1 < self.attempts:
+                    time.sleep(2 ** attempt)
+        raise BackendError(f"{url}: failed after {self.attempts} attempts ({last})") from last
+
+
+@dataclass
+class CommandBackend(Backend):
+    """Runs a command and reads JSON from its stdout.
+
+    The escape hatch that makes the language of your retrieval system irrelevant:
+
+        rag-eval-gate eval gold.jsonl \\
+            --backend command \\
+            --command './my-retriever --query {query} --mode {mode} --json'
+
+    `{query}` and `{mode}` are substituted as single shell-quoted arguments, so a query
+    containing spaces, quotes or a semicolon is passed through as one argument rather than
+    reinterpreted by the shell.
+    """
+
+    command_template: str
+    results_path: str = "results"
+    id_field: str = "docId"
+    score_field: str | None = "score"
+    timeout: float = 120.0
+
+    def search(self, query: str, mode: str) -> list[Hit]:
+        command = self.command_template.format(query=shlex.quote(query),
+                                               mode=shlex.quote(mode))
+        try:
+            finished = subprocess.run(shlex.split(command), capture_output=True, text=True,
+                                      timeout=self.timeout, check=False)
+        except (OSError, subprocess.TimeoutExpired) as e:
+            raise BackendError(f"{command}: {e}") from e
+        if finished.returncode != 0:
+            tail = (finished.stderr or "").strip()[-400:]
+            raise BackendError(f"exit {finished.returncode}: {command}\n{tail}")
+        try:
+            payload = json.loads(finished.stdout)
+        except json.JSONDecodeError as e:
+            raise BackendError(f"{command}: stdout is not JSON ({e})") from e
+        rows = _dig(payload, self.results_path)
+        http = HttpBackend("", self.results_path, self.id_field, self.score_field)
+        return [http._hit(row, command) for row in rows]
+
+
+@dataclass
+class RunFileBackend(Backend):
+    """A TREC run file: `query_id Q0 doc_id rank score run_name`, whitespace separated.
+
+    The format the IR community already exchanges results in, so a corpus can be ranked
+    once — offline, on a GPU, in whatever language — and evaluated as often as needed with
+    nothing running. Queries are addressed by id, so the gold set must supply ids too;
+    `--mode` is ignored, since a run file holds exactly one system's output.
+
+    Lines are sorted by the rank column when present, and otherwise kept in file order.
+    """
+
+    path: str | Path
+    _by_query: dict[str, list[Hit]] = field(init=False, default_factory=dict)
+
+    def __post_init__(self) -> None:
+        try:
+            text = Path(self.path).read_text(encoding="utf-8")
+        except OSError as e:
+            # Surfaced as a backend error so the CLI exits 2 (a usage problem) rather than
+            # 1 (a gate breach) — the two mean very different things to a CI job.
+            raise BackendError(f"cannot read run file {self.path}: {e}") from e
+        staged: dict[str, list[tuple[int, Hit]]] = {}
+        for number, line in enumerate(text.splitlines(), 1):
+            parts = line.split()
+            if not parts or line.lstrip().startswith("#"):
+                continue
+            if len(parts) < 3:
+                raise BackendError(f"{self.path}:{number}: expected at least 3 columns")
+            query_id, doc_id = parts[0], parts[2]
+            rank = _maybe_int(parts[3]) if len(parts) > 3 else None
+            score = _maybe_float(parts[4]) if len(parts) > 4 else None
+            staged.setdefault(query_id, []).append(
+                (rank if rank is not None else len(staged.get(query_id, [])), Hit(doc_id, score)))
+        self._by_query = {q: [hit for _, hit in sorted(rows, key=lambda r: r[0])]
+                          for q, rows in staged.items()}
+
+    def search(self, query: str, mode: str) -> list[Hit]:
+        if query not in self._by_query:
+            raise BackendError(f"{self.path}: no rankings for query id '{query}'. "
+                               "A run file is addressed by query id — does the gold set "
+                               "supply ids matching it?")
+        return self._by_query[query]
+
+    @property
+    def query_ids(self) -> list[str]:
+        return list(self._by_query)
+
+
+def _maybe_int(text: str) -> int | None:
+    try:
+        return int(text)
+    except ValueError:
+        return None
+
+
+def _maybe_float(text: str) -> float | None:
+    try:
+        return float(text)
+    except ValueError:
+        return None
+
+
+def build_backend(kind: str, **options) -> Backend:
+    """Construct a backend from CLI-shaped options, ignoring the ones it does not take."""
+    if kind == "http":
+        return HttpBackend(
+            url_template=options["url_template"],
+            results_path=options.get("results_path", "results"),
+            id_field=options.get("id_field", "docId"),
+            score_field=options.get("score_field", "score"),
+            timeout=options.get("timeout", 120.0),
+        )
+    if kind == "command":
+        return CommandBackend(
+            command_template=options["command"],
+            results_path=options.get("results_path", "results"),
+            id_field=options.get("id_field", "docId"),
+            score_field=options.get("score_field", "score"),
+            timeout=options.get("timeout", 120.0),
+        )
+    if kind == "run-file":
+        return RunFileBackend(options["run_file"])
+    raise BackendError(f"unknown backend '{kind}' — expected http, command or run-file")