evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/reasoning/termination_audit.py

@@ -0,0 +1,258 @@
+"""Termination audit — *why* did the generation stop?
+
+A missing answer has at least four unrelated causes: the decode hit the token
+budget (truncated), the model explicitly declined (gave up), it fell into a
+repetition loop (degenerate), or it finished cleanly and was simply wrong.
+These need different fixes — a bigger budget, a prompt change, a decoding
+change, a capability change — and only the last one is a reasoning failure.
+
+This is a **confound control**, not a headline metric.  Truncated and degenerate
+cases carry short, tag-less outputs, which is exactly the signature that
+correlates with *every* other probe's "bad" column, so leaving them in the pool
+makes M2 attribute a decoding-budget problem to whatever mechanism is being
+tested.  Stratify on ``termination_class`` before reading any other probe.
+
+One optional generation per non-clean case continues the truncated text: if the
+continuation lands on an answer, the case was budget-limited, not stuck.
+
+References:
+- The Curious Case of Neural Text Degeneration — Holtzman et al., ICLR 2020 —
+  arXiv:1904.09751 (repetition as a decoding pathology, not a knowledge one)
+- Measuring Faithfulness in Chain-of-Thought Reasoning — Lanham et al., 2023 —
+  arXiv:2307.13702 (truncated-reasoning protocol)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import (
+    answer_equal,
+    extract_answer,
+    has_answer_tag,
+    looks_like_give_up,
+    looks_truncated,
+    repetition_score,
+)
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_CONTINUE_SUFFIX = (
+    "Continue the reasoning above from where it stops and finish it. "
+    "End with the final answer on its own last line as 'Answer: <answer>'."
+)
+
+CLEAN = "clean"
+TRUNCATED = "truncated"
+DEGENERATE = "degenerate"
+GAVE_UP = "gave_up"
+NO_ANSWER = "no_answer"
+
+
+@register_analyzer("termination_audit")
+class TerminationAudit(Analyzer):
+    """Classify how each generation ended and test whether continuing rescues it.
+
+    Hyper-parameters:
+        continue_non_clean: spend one generation continuing each non-clean case.
+        repetition_n:       word n-gram width for the degeneration score.
+        repetition_thresh:  repetition_score above this ⇒ ``degenerate``.
+        max_cases:          label-stratified cap; 0 (the default) = every case.
+        answer_fn/match_fn: answer extraction / equality (see ``_text``).
+    """
+
+    name = "termination_audit"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+    signal_docs = {
+        'clean_rate': ('Finished normally', 'Share of answers that finished normally.'),
+        'continuation_chars': 'How much more the model wrote when allowed to continue.',
+        'continuation_has_answer': 'Whether letting a cut-off answer continue produced an answer.',
+        'degenerate_rate': ('Fell into repeating', 'Share of answers that fell into repeating themselves.'),
+        'gave_up': ('Model gave up', 'Whether the model said it could not answer.'),
+        'giveup_rate': ('Gave up', 'Share of answers where the model declined to answer.'),
+        'has_answer_tag': 'Whether the model used the answer format it was asked for.',
+        'looks_truncated': ('Answer was cut off', 'Whether the answer appears to have been cut off mid-thought.'),
+        'matches_output_contract': 'Whether the answer is in the shape the task asked for.',
+        'n_cases': 'How many answers were inspected.',
+        'non_clean_rate': ('Did not finish cleanly', 'Share of answers that did NOT finish normally — cut off, repeating, or giving up.'),
+        'output_chars': ('Answer length', 'How long the answer was, in characters.'),
+        'output_words': ('Answer length (words)', 'How long the answer was, in words.'),
+        'recovered_rate': 'Share of cut-off answers that did produce an answer when allowed to continue.',
+        'repetition_score': ('Repetitiveness', 'How much the answer repeats itself. High means the model was looping.'),
+        'termination_class': 'How this answer ended: finished, cut off, repeating, or gave up.',
+        'truncation_rate': ('Cut off by length', 'Share of answers cut off by the length limit before finishing.'),
+    }
+
+    def __init__(
+        self,
+        continue_non_clean: bool = True,
+        repetition_n: int = 8,
+        repetition_thresh: float = 0.5,
+        max_cases: int = 0,
+        answer_fn: Optional[Callable[[Any], str]] = None,
+        match_fn: Optional[Callable[[Any, Any], bool]] = None,
+    ) -> None:
+        super().__init__(
+            continue_non_clean=continue_non_clean,
+            repetition_n=repetition_n,
+            repetition_thresh=repetition_thresh,
+            max_cases=max_cases,
+        )
+        self.answer_fn = answer_fn or extract_answer
+        self.match_fn = match_fn or answer_equal
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            text = case.observed
+            if not str(text or "").strip():
+                # no stored output — generate one so the class is about THIS model
+                text = str(model.generate(case.inputs))
+            entry = self._classify(case, text)
+            if self.continue_non_clean and entry["termination_class"] != CLEAN:
+                entry.update(self._continue(model, case, text))
+            per_case.append(entry)
+
+        n = len(per_case)
+        counts = {
+            cls: sum(1 for c in per_case if c["termination_class"] == cls)
+            for cls in (CLEAN, TRUNCATED, DEGENERATE, GAVE_UP, NO_ANSWER)
+        }
+        rescued = [c for c in per_case if "recovered_by_continuation" in c]
+        findings: dict[str, Any] = {
+            "n_cases": n,
+            "class_counts": counts,
+            "clean_rate": round(counts[CLEAN] / n, 4) if n else None,
+            "non_clean_rate": round(1 - counts[CLEAN] / n, 4) if n else None,
+            "truncation_rate": round(counts[TRUNCATED] / n, 4) if n else None,
+            "degenerate_rate": round(counts[DEGENERATE] / n, 4) if n else None,
+            "giveup_rate": round(counts[GAVE_UP] / n, 4) if n else None,
+            "recovered_rate": (
+                round(sum(c["recovered_by_continuation"] for c in rescued) / len(rescued), 4)
+                if rescued
+                else None
+            ),
+            "per_case": per_case,
+            "_caveat": (
+                "CONFOUND CONTROL — read this before any other probe. Cases with "
+                "termination_class != 'clean' have short, tag-less outputs, the "
+                "same signature every other probe scores as 'bad'; leaving them "
+                "pooled makes M2 attribute a decoding-budget or repetition "
+                "problem to whichever mechanism is under test. Stratify on "
+                "termination_class (or exclude non-clean cases) first. A high "
+                "recovered_by_continuation rate means the budget is the defect: "
+                "raise max_tokens and re-collect before diagnosing anything else. "
+                "The continuation columns are INTERVENTIONAL — held-out "
+                "verification must re-run them."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _classify(self, case: "FailureCase", text: Any) -> dict[str, Any]:
+        raw = str(text or "")
+        repetition = repetition_score(raw, self.repetition_n)
+        tagged = has_answer_tag(raw)
+        truncated = looks_truncated(raw)
+        gave_up = looks_like_give_up(raw)
+
+        # A bare short answer with no terminal punctuation and no <answer>/
+        # \boxed{} tag ("guitar", "two", "yes") is exactly what looks_truncated
+        # flags -- correctly, when it's a guess from text shape alone. But a
+        # short-answer dataset (VQA-style single word/number gold answers)
+        # produces that EXACT shape on every case that terminated perfectly
+        # normally. When the model itself has told us how generation actually
+        # ended (metadata["finish_reason"], populated by e.g. mine_cases.py's
+        # generate_with_meta -- see fix_agent.py's L0 telemetry gate for the
+        # other consumer of this same field), that real signal overrides the
+        # text-shape guess: "stop" means it did NOT hit the token budget,
+        # regardless of how terse the output looks; "length" confirms it did,
+        # even if the text happens to end in punctuation. No finish_reason on
+        # the case (API models, or examples that never wired mine_cases.py's
+        # meta path) falls back to the heuristic, unchanged from before.
+        finish_reason = (getattr(case, "metadata", {}) or {}).get("finish_reason")
+        if finish_reason == "stop":
+            truncated = False
+        elif finish_reason == "length":
+            truncated = True
+
+        contract = (getattr(case, "metadata", {}) or {}).get("output_contract") or {}
+        contract_match = _matches_output_contract(raw, contract)
+        if contract_match:
+            truncated = False
+
+        # Order matters: degeneration is checked first because a looping
+        # generation also LOOKS truncated (it is cut off mid-loop), and the fix
+        # for a loop is a decoding change, not a bigger budget.
+        if repetition >= self.repetition_thresh:
+            cls = DEGENERATE
+        elif gave_up:
+            cls = GAVE_UP
+        elif truncated:
+            cls = TRUNCATED
+        elif tagged or contract_match:
+            cls = CLEAN
+        else:
+            cls = NO_ANSWER
+        return {
+            "sample_id": case.id,
+            "termination_class": cls,
+            "repetition_score": repetition,
+            "has_answer_tag": int(tagged),
+            "looks_truncated": int(truncated),
+            "gave_up": int(gave_up),
+            "matches_output_contract": int(contract_match),
+            "output_chars": len(raw),
+            "output_words": len(raw.split()),
+        }
+
+    def _continue(self, model: "Model", case: "FailureCase", text: Any) -> dict[str, Any]:
+        prompt = (
+            f"{case.inputs.prompt or ''}\n\nPartial response:\n{text}\n\n{_CONTINUE_SUFFIX}"
+        )
+        continuation = str(model.generate(dataclasses.replace(case.inputs, prompt=prompt)))
+        out: dict[str, Any] = {
+            "continuation_has_answer": int(has_answer_tag(continuation)),
+            "continuation_chars": len(continuation),
+        }
+        graded = (
+            self.match_fn(self.answer_fn(continuation), case.expected)
+            if case.expected is not None
+            else None
+        )
+        # "Recovered" means the model could finish given more room: it produced
+        # an answer AND — when a gold exists — the right one.
+        out["recovered_by_continuation"] = int(
+            out["continuation_has_answer"] == 1 and graded is not False
+        )
+        if graded is not None:
+            out["continuation_correct"] = int(graded)
+        return out
+
+
+def _matches_output_contract(raw: str, contract: Any) -> bool:
+    """Return whether a terse response is complete under a declared contract."""
+    if not isinstance(contract, dict):
+        return False
+    text = str(raw or "").strip()
+    if not text:
+        return False
+    if contract.get("kind") == "multiple_choice_letter":
+        choices = {str(v).strip().upper() for v in contract.get("choices", [])}
+        return bool(choices) and text.upper() in choices
+    pattern = contract.get("pattern")
+    if isinstance(pattern, str) and pattern:
+        try:
+            return re.fullmatch(pattern, text) is not None
+        except re.error:
+            return False
+    return False

evalrx/analyzers/uncertainty/__init__.py

@@ -0,0 +1,18 @@
+"""Uncertainty analyzers — cheap, mostly black-box signals of model (un)certainty."""
+
+from evalrx.analyzers.uncertainty.calibration import CalibrationAnalyzer
+from evalrx.analyzers.uncertainty.coverage_gap import CoverageVerificationGap
+from evalrx.analyzers.uncertainty.entropy import TokenEntropyAnalyzer, UncertaintyResult
+from evalrx.analyzers.uncertainty.logprob_entropy import LogprobEntropyAnalyzer
+from evalrx.analyzers.uncertainty.self_consistency import SelfConsistencyAnalyzer
+from evalrx.analyzers.uncertainty.verbalized_conf import VerbalizedConfidenceAnalyzer
+
+__all__ = [
+    "TokenEntropyAnalyzer",
+    "UncertaintyResult",
+    "LogprobEntropyAnalyzer",
+    "SelfConsistencyAnalyzer",
+    "VerbalizedConfidenceAnalyzer",
+    "CalibrationAnalyzer",
+    "CoverageVerificationGap",
+]

evalrx/analyzers/uncertainty/calibration.py

@@ -0,0 +1,174 @@
+"""Confidence calibration — ECE / Brier / overconfidence gap against case labels.
+
+Where :class:`LogprobEntropyAnalyzer` and :class:`VerbalizedConfidenceAnalyzer`
+emit raw per-case uncertainty, this analyzer closes the loop against PASS/FAIL
+labels: it bins confidence, compares each bin's stated confidence with its
+actual accuracy, and reports expected calibration error plus the signed
+overconfidence gap — for BOTH confidence channels (sequence logprob and
+verbalized).  A model that is wrong exactly where it is confident needs a
+different fix (abstention / recalibration) than one that is merely inaccurate.
+
+``requires=GENERATE``: the verbalized channel runs on every model; the logprob
+channel needs :attr:`Capability.LOGPROBS` and is reported empty (``n=0``,
+``logprob_channel_available=False``) on a model without it -- an API model
+that returns no logprobs (Gemini) keeps its calibration signal instead of
+losing the whole analyzer. Labels come from the batch, unlabeled cases are
+skipped.
+
+References:
+- On Calibration of Modern Neural Networks — Guo et al., ICML 2017 —
+  arXiv:1706.04599 (ECE)
+- Just Ask for Calibration — Tian et al., EMNLP 2023 — arXiv:2305.14975
+- Language Models (Mostly) Know What They Know — Kadavath et al., 2022 —
+  arXiv:2207.05221
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import math
+import re
+from typing import TYPE_CHECKING, Any, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.case import CaseBatch, Label
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.model import Model
+
+_ELICIT = (
+    "\n\nAfter answering, state your confidence on a 0-100 scale "
+    "on its own line as: Confidence: <number>"
+)
+_CONF = re.compile(r"confidence[:=]?\s*(\d{1,3}(?:\.\d+)?)\s*%?", re.IGNORECASE)
+
+
+def _parse_conf(text: str) -> Optional[float]:
+    m = _CONF.search(str(text or ""))
+    if not m:
+        return None
+    raw = m.group(1)
+    val = float(raw)
+    if val > 1.0:
+        return max(0.0, min(1.0, val / 100.0))
+    # the elicit asks for a 0-100 scale: an integer 0/1 is a percent, while a
+    # decimal like 0.8 is taken as an (off-instruction but common) fraction
+    return max(0.0, min(1.0, val if "." in raw else val / 100.0))
+
+
+def expected_calibration_error(pairs: list[tuple[float, bool]], n_bins: int) -> Optional[float]:
+    """Standard binned ECE over ``(confidence, correct)`` pairs."""
+    if not pairs:
+        return None
+    bins: list[list[tuple[float, bool]]] = [[] for _ in range(n_bins)]
+    for conf, correct in pairs:
+        index = min(n_bins - 1, int(conf * n_bins))
+        bins[index].append((conf, correct))
+    total = len(pairs)
+    ece = 0.0
+    for bucket in bins:
+        if not bucket:
+            continue
+        mean_conf = sum(c for c, _ in bucket) / len(bucket)
+        accuracy = sum(1 for _, ok in bucket if ok) / len(bucket)
+        ece += (len(bucket) / total) * abs(mean_conf - accuracy)
+    return round(ece, 4)
+
+
+@register_analyzer("calibration")
+class CalibrationAnalyzer(Analyzer):
+    """Label-anchored calibration: ECE, Brier and overconfidence gap for logprob and verbalized confidence.
+
+    Hyper-parameters:
+        n_bins:    ECE bins.
+        max_cases: label-stratified cap (one logprobs + one generate per case); 0 (the default) = every case.
+        elicit:    verbalized-confidence suffix appended to the prompt.
+    """
+
+    name = "calibration"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(self, n_bins: int = 10, max_cases: int = 0, elicit: str = _ELICIT) -> None:
+        super().__init__(n_bins=max(2, int(n_bins)), max_cases=max_cases, elicit=elicit)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        lp_pairs: list[tuple[float, bool]] = []
+        vb_pairs: list[tuple[float, bool]] = []
+        labelled = CaseBatch([c for c in cases if c.label in (Label.PASS, Label.FAIL)])
+        n_unlabelled = len(list(cases)) - len(list(labelled))
+        has_logprobs = Capability.LOGPROBS in getattr(model, "capabilities", frozenset())
+        for case in labelled.stratified_head(self.max_cases):
+            correct = case.label == Label.PASS
+            entry: dict[str, Any] = {"sample_id": case.id, "correct": int(correct)}
+
+            # An API timeout is evidence missing for one case, not a reason to
+            # discard an otherwise useful calibration run.  In particular,
+            # Gemini's transport can time out a single multimodal request while
+            # the next clip succeeds.  Keep the failed case visible and carry
+            # on; summary denominators below then honestly reflect only the
+            # successful confidence measurements.
+            try:
+                toks = model.logprobs(case.inputs) if has_logprobs else []
+            except Exception as exc:
+                toks = []
+                entry["logprob_error"] = f"{type(exc).__name__}: {str(exc)[:160]}"
+            lps = [t.logprob for t in toks]
+            conf_lp = math.exp(sum(lps) / len(lps)) if lps else None
+            if conf_lp is not None:
+                conf_lp = max(0.0, min(1.0, conf_lp))
+                entry["conf_logprob"] = round(conf_lp, 4)
+                lp_pairs.append((conf_lp, correct))
+
+            elicited = dataclasses.replace(
+                case.inputs, prompt=(case.inputs.prompt or "") + self.elicit
+            )
+            try:
+                raw = str(model.generate(elicited))
+            except Exception as exc:
+                entry["verbalized_error"] = f"{type(exc).__name__}: {str(exc)[:160]}"
+                per_case.append(entry)
+                continue
+            conf_vb = _parse_conf(raw)
+            if conf_vb is not None:
+                entry["conf_verbal"] = round(conf_vb, 4)
+                vb_pairs.append((conf_vb, correct))
+            per_case.append(entry)
+
+        def _summary(pairs: list[tuple[float, bool]]) -> dict[str, Any]:
+            if not pairs:
+                return {"n": 0, "ece": None, "brier": None, "overconfidence_gap": None}
+            accuracy = sum(1 for _, ok in pairs if ok) / len(pairs)
+            mean_conf = sum(c for c, _ in pairs) / len(pairs)
+            brier = sum((c - (1.0 if ok else 0.0)) ** 2 for c, ok in pairs) / len(pairs)
+            return {
+                "n": len(pairs),
+                "ece": expected_calibration_error(pairs, self.n_bins),
+                "brier": round(brier, 4),
+                "overconfidence_gap": round(mean_conf - accuracy, 4),
+            }
+
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_unlabelled_skipped": n_unlabelled,
+            "n_bins": self.n_bins,
+            "logprob_channel_available": has_logprobs,
+            "logprob_channel": _summary(lp_pairs),
+            "verbalized_channel": _summary(vb_pairs),
+            "per_case": per_case,
+            "_caveat": (
+                "Needs PASS/FAIL labels — unlabeled cases are skipped, and ECE "
+                "on few labelled cases is unstable (aim for >= ~50). "
+                "conf_logprob is the geometric-mean token probability of the "
+                "model's own continuation — a proxy, not an answer probability; "
+                "on API backends it reflects only the returned top-k, and a model "
+                "without LOGPROBS reports only the verbalized channel. A positive "
+                "overconfidence_gap on FAIL-heavy slices argues for abstention/"
+                "recalibration fixes rather than capability fixes."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)

evalrx/analyzers/uncertainty/coverage_gap.py

@@ -0,0 +1,199 @@
+"""Coverage-vs-verification gap — did the model *have* the answer and fail to pick it?
+
+pass@k and accuracy answer different questions, and the difference decides the
+fix tier.  If the gold answer appears in *some* of k samples but the majority
+vote lands elsewhere, the capability is present and the SELECTION is broken —
+a reranker, a verifier, or a better aggregation (L1/L2) recovers it outright.
+If no sample is ever right, no amount of scaffolding helps and the fix has to
+change the model (L3/L4).  Reporting one accuracy number cannot tell these
+apart, and the second is by far the more expensive mistake.
+
+The optional self-verification arm goes one step further: it hands the model
+its own k candidates and asks it to choose.  The distance between ``pass_at_k``
+and ``verify_correct`` is the headroom a self-verifier can actually reach, as
+opposed to the headroom that exists in principle.
+
+Cost: k generations per case (+1 with ``self_verify=True``).
+
+References:
+- Evaluating Large Language Models Trained on Code — Chen et al., 2021 —
+  arXiv:2107.03374 (the unbiased pass@k estimator)
+- Self-Consistency Improves Chain of Thought Reasoning — Wang et al., ICLR 2023
+  — arXiv:2203.11171 (majority@k)
+- Large Language Monkeys: Scaling Inference Compute with Repeated Sampling —
+  Brown et al., 2024 — arXiv:2407.21787 (coverage vs verification framing)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import math
+from collections import Counter
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import (
+    answer_equal,
+    extract_answer,
+    normalize_answer,
+)
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_VERIFY = (
+    "Here are candidate answers produced for the question above:\n\n{candidates}\n\n"
+    "Decide which one is correct. Give only that answer on its own last line as "
+    "'Answer: <answer>'."
+)
+
+
+def pass_at_k(n: int, c: int, k: int) -> float:
+    """Unbiased pass@k for *c* correct out of *n* samples (Chen et al., 2021)."""
+    if n - c < k:
+        return 1.0
+    return round(1.0 - math.comb(n - c, k) / math.comb(n, k), 4)
+
+
+@register_analyzer("coverage_verification_gap")
+class CoverageVerificationGap(Analyzer):
+    """Sample k answers and split "cannot solve" from "cannot select".
+
+    Hyper-parameters:
+        k:           samples per case.
+        self_verify: +1 generation asking the model to choose among its own k.
+        max_cases:   label-stratified cap (k generations each); 0 (the default) = every case.
+        gen_kwargs:  passed to ``model.generate`` — REQUIRES temperature > 0;
+                     at temperature 0 all k samples are identical and the gap is
+                     structurally 0.
+        grader:      ``callable(prediction, case) -> bool | None`` — injected the
+                     same way as the agent-side ``reliability_probe``.
+        answer_fn:   ``callable(text) -> str`` answer extractor.
+    """
+
+    name = "coverage_verification_gap"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        k: int = 5,
+        self_verify: bool = False,
+        max_cases: int = 0,
+        gen_kwargs: Optional[dict] = None,
+        grader: Optional[Callable[[Any, "FailureCase"], Optional[bool]]] = None,
+        answer_fn: Optional[Callable[[Any], str]] = None,
+    ) -> None:
+        super().__init__(
+            k=max(int(k), 1),
+            self_verify=self_verify,
+            max_cases=max_cases,
+            gen_kwargs=dict(gen_kwargs or {}),
+        )
+        # ctor names, so sklearn-style get_params() reflection works
+        self.grader = grader or _default_grader
+        self.answer_fn = answer_fn or extract_answer
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            if case.expected is None:
+                per_case.append({"sample_id": case.id, "skipped": "no gold answer"})
+                continue
+            per_case.append(self._probe_case(model, case))
+
+        scored = [c for c in per_case if "coverage_gap" in c]
+        degenerate = bool(scored) and all(c["n_unique"] == 1 for c in scored)
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(scored),
+            "k": self.k,
+            "gen_kwargs": dict(self.gen_kwargs),
+            "mean_pass_at_k": _mean([c["pass_at_k"] for c in scored]),
+            "mean_majority_correct": _mean([c["majority_correct"] for c in scored]),
+            "coverage_gap_rate": _mean([c["coverage_gap"] for c in scored]),
+            "mean_verify_correct": _mean(
+                [c["verify_correct"] for c in scored if "verify_correct" in c]
+            ),
+            "no_coverage_rate": _mean([1 - c["pass_at_k"] for c in scored]),
+            "degenerate_sampling": degenerate,
+            "per_case": per_case,
+            "_caveat": (
+                "coverage_gap = 'the answer was in the pool, the vote missed it' "
+                "⇒ a selection defect, fixable at L1/L2 with a verifier or "
+                "reranker. no_coverage_rate = 'never produced once in k' ⇒ a "
+                "capability defect no scaffold reaches; do not let a headline "
+                "pass@k hide which of the two dominates. REQUIRES sampling: "
+                "degenerate_sampling=true means every sample was identical "
+                "(temperature 0 or a deterministic backend) and the gap is "
+                "structurally zero, not measured. pass_at_k here uses the "
+                "unbiased estimator over exactly k draws, so with k small its "
+                "variance is large per case — read the batch rate. "
+                "INTERVENTIONAL: held-out confirmation must re-sample."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase") -> dict[str, Any]:
+        samples = [
+            str(model.generate(case.inputs, **self.gen_kwargs)) for _ in range(self.k)
+        ]
+        answers = [self.answer_fn(s) for s in samples]
+        grades = [self.grader(s, case) for s in samples]
+        graded = [g for g in grades if g is not None]
+        entry: dict[str, Any] = {"sample_id": case.id, "n_samples": len(samples)}
+        if not graded:
+            entry["skipped"] = "no sample could be graded"
+            return entry
+
+        n_correct = sum(graded)
+        counts = Counter(normalize_answer(a) for a in answers)
+        modal, modal_n = counts.most_common(1)[0]
+        modal_correct = bool(answer_equal(modal, case.expected))
+
+        entry.update(
+            {
+                "n_graded": len(graded),
+                "n_correct": n_correct,
+                "n_unique": len(counts),
+                "pass_at_k": pass_at_k(len(graded), n_correct, len(graded)),
+                "any_correct": int(n_correct > 0),
+                "majority_correct": int(modal_correct),
+                "majority_share": round(modal_n / len(answers), 4),
+                # the diagnostic column: present in the pool, missed by the vote
+                "coverage_gap": int(n_correct > 0 and not modal_correct),
+                "modal_answer": str(modal)[:80],
+            }
+        )
+        if self.self_verify:
+            listing = "\n".join(
+                f"{i + 1}. {a}" for i, a in enumerate(dict.fromkeys(answers))
+            )
+            chosen = model.generate(
+                dataclasses.replace(
+                    case.inputs,
+                    prompt=f"{case.inputs.prompt or ''}\n\n{_VERIFY.format(candidates=listing)}",
+                )
+            )
+            verdict = self.grader(chosen, case)
+            if verdict is not None:
+                entry["verify_correct"] = int(verdict)
+                # how much of the available headroom self-verification captured
+                entry["verifier_gap"] = int(n_correct > 0 and not verdict)
+        return entry
+
+
+def _mean(values: list) -> Optional[float]:
+    clean = [v for v in values if v is not None]
+    return round(sum(clean) / len(clean), 4) if clean else None
+
+
+def _default_grader(prediction: Any, case: "FailureCase") -> Optional[bool]:
+    if case.expected is None:
+        return None
+    return answer_equal(extract_answer(prediction), case.expected)