evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/reasoning/contamination.py

@@ -0,0 +1,214 @@
+"""Contamination probe — is this benchmark measuring reasoning or recall?
+
+Every accuracy number in M1/M2 assumes the model is solving the item, not
+remembering it.  When a benchmark leaked into pre-training, PASS/FAIL stops
+tracking capability, the mechanism columns correlate with nothing real, and any
+fix validated on it will not transfer.  Contamination is therefore a
+**confound column**, not a headline: it is measured per case, carried into M2
+alongside the mechanism columns, and used to disqualify a finding whose effect
+lives entirely on memorised items.
+
+Needs no labels and no reference corpus.  The guided-vs-general design of
+Golchin & Surdeanu isolates the effect: both arms ask the model to complete the
+second half of the item, but only the guided arm names the benchmark and split.
+A model that reconstructs the continuation *better when told which benchmark it
+came from* is retrieving, not generalising.
+
+Cost: 2 generations per case (3 with ``ask_provenance=True``).
+
+References:
+- Time Travel in LLMs: Tracing Data Contamination in Large Language Models —
+  Golchin & Surdeanu, ICLR 2024 — arXiv:2308.08493 (guided vs general prompting)
+- Investigating Data Contamination in Modern Benchmarks for LLMs —
+  Deng et al., NAACL 2024 — arXiv:2311.09783
+- Rethinking Benchmark and Contamination for Language Models with
+  Rephrased Samples — Yang et al., 2023 — arXiv:2311.04850
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, Any, Optional
+
+from evalrx.analyzers.reasoning._text import word_ngrams
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_GENERAL = (
+    "Complete the rest of this text exactly as it continues. Output only the "
+    "continuation.\n\nText so far:\n{first_half}"
+)
+_GUIDED = (
+    "The text below is the FIRST PIECE of an instance from the {split} split of "
+    "the {dataset} dataset. Finish the second piece exactly as it appears in the "
+    "dataset. Output only the continuation.\n\nFirst piece:\n{first_half}"
+)
+_PROVENANCE = (
+    "Which public benchmark or dataset is the following item from? Answer with "
+    "the dataset name only, or 'unknown'.\n\n{text}"
+)
+
+
+def overlap_score(candidate: Any, reference: Any, n: int = 4) -> float:
+    """Fraction of the reference's word ``n``-grams that the candidate reproduces.
+
+    Recall-oriented on purpose: a verbose completion that contains the reference
+    span has still reproduced it, and length-penalising that would hide exactly
+    the memorisation this probe is looking for.  Falls back to unigrams for
+    references too short to have ``n``-grams.
+    """
+    ref_grams = word_ngrams(reference, n) or word_ngrams(reference, 1)
+    if not ref_grams:
+        return 0.0
+    width = len(ref_grams[0])
+    cand_grams = set(word_ngrams(candidate, width))
+    if not cand_grams:
+        return 0.0
+    hits = sum(1 for g in ref_grams if g in cand_grams)
+    return round(hits / len(ref_grams), 4)
+
+
+@register_analyzer("contamination_score")
+class ContaminationProbe(Analyzer):
+    """Guided-vs-general completion overlap — a per-case memorisation signal.
+
+    Hyper-parameters:
+        dataset_name:    benchmark name used by the guided arm (REQUIRED to be
+                         meaningful — the guided/general contrast IS the design).
+        split:           split name used by the guided arm.
+        split_frac:      fraction of the item shown as the first half.
+        ngram:           n-gram width for the overlap score.
+        flag_threshold:  guided overlap above this ⇒ ``verbatim_flag``.
+        ask_provenance:  +1 generation asking the model to name the benchmark.
+        max_cases:       label-stratified cap; 0 (the default) = every case.
+    """
+
+    name = "contamination_score"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        dataset_name: str = "the evaluation",
+        split: str = "test",
+        split_frac: float = 0.5,
+        ngram: int = 4,
+        flag_threshold: float = 0.6,
+        ask_provenance: bool = False,
+        max_cases: int = 0,
+    ) -> None:
+        super().__init__(
+            dataset_name=dataset_name,
+            split=split,
+            split_frac=min(max(split_frac, 0.1), 0.9),
+            ngram=ngram,
+            flag_threshold=flag_threshold,
+            ask_provenance=ask_provenance,
+            max_cases=max_cases,
+        )
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            per_case.append(self._probe_case(model, case))
+
+        scored = [c for c in per_case if "guided_overlap" in c]
+        flagged = [c for c in scored if c["verbatim_flag"] == 1]
+        labelled = [c for c in scored if c.get("labelled_pass") is not None]
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(scored),
+            "dataset_name": self.dataset_name,
+            "mean_general_overlap": _mean([c["general_overlap"] for c in scored]),
+            "mean_guided_overlap": _mean([c["guided_overlap"] for c in scored]),
+            "mean_guided_gain": _mean([c["guided_gain"] for c in scored]),
+            "verbatim_flag_rate": (
+                round(len(flagged) / len(scored), 4) if scored else None
+            ),
+            # the number that actually matters: does memorisation buy accuracy?
+            "accuracy_on_flagged": _mean([c["labelled_pass"] for c in labelled
+                                          if c["verbatim_flag"] == 1]),
+            "accuracy_on_unflagged": _mean([c["labelled_pass"] for c in labelled
+                                            if c["verbatim_flag"] == 0]),
+            "per_case": per_case,
+            "_caveat": (
+                "This is a CONFOUND column, never a headline. A guided_gain > 0 "
+                "says the model reconstructs the item better when told which "
+                "benchmark it is from — evidence of exposure, not proof, and it "
+                "is confounded by items whose continuation is guessable from "
+                "format alone (templated benchmarks score high with no leak, so "
+                "read general_overlap as the floor). The usable test is the "
+                "SPLIT: when accuracy_on_flagged far exceeds "
+                "accuracy_on_unflagged, the benchmark's PASS mass is partly "
+                "recall and any M3 hypothesis resting on it must be re-run on "
+                "unflagged cases. Set dataset_name — with the default the "
+                "guided arm names nothing and the contrast is void."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase") -> dict[str, Any]:
+        from evalrx.core.case import Label
+
+        text = str(case.inputs.prompt or "")
+        words = text.split()
+        entry: dict[str, Any] = {"sample_id": case.id, "n_words": len(words)}
+        if len(words) < 2 * self.ngram:
+            entry["skipped"] = "prompt too short to split into two halves"
+            return entry
+
+        cut = max(1, int(len(words) * self.split_frac))
+        first_half, second_half = " ".join(words[:cut]), " ".join(words[cut:])
+
+        general = str(model.generate(_bare(case, _GENERAL.format(first_half=first_half))))
+        guided = str(
+            model.generate(
+                _bare(
+                    case,
+                    _GUIDED.format(
+                        first_half=first_half,
+                        dataset=self.dataset_name,
+                        split=self.split,
+                    ),
+                )
+            )
+        )
+        general_overlap = overlap_score(general, second_half, self.ngram)
+        guided_overlap = overlap_score(guided, second_half, self.ngram)
+        entry.update(
+            {
+                "general_overlap": general_overlap,
+                "guided_overlap": guided_overlap,
+                "guided_gain": round(guided_overlap - general_overlap, 4),
+                "verbatim_flag": int(guided_overlap >= self.flag_threshold),
+                "labelled_pass": (
+                    int(case.label == Label.PASS)
+                    if case.label in (Label.PASS, Label.FAIL)
+                    else None
+                ),
+            }
+        )
+        if self.ask_provenance:
+            named = str(model.generate(_bare(case, _PROVENANCE.format(text=text[:1500]))))
+            entry["named_dataset"] = named.strip()[:80]
+            entry["named_correct"] = int(
+                self.dataset_name.lower().split()[0] in named.lower()
+            )
+        return entry
+
+
+def _bare(case: "FailureCase", prompt: str):
+    """Inputs carrying *prompt* alone — media slots are preserved, the task is not."""
+    return dataclasses.replace(case.inputs, prompt=prompt)
+
+
+def _mean(values: list) -> Optional[float]:
+    clean = [v for v in values if v is not None]
+    return round(sum(clean) / len(clean), 4) if clean else None

evalrx/analyzers/reasoning/knowledge_split.py

@@ -0,0 +1,253 @@
+"""Knowledge/reasoning split — is the fact missing, or is the chaining broken?
+
+A wrong multi-hop answer has two incompatible causes.  Either the model never
+knew the bridging fact (a *knowledge* deficit — retrieval, tools, or a bigger
+model), or it knew every fact and still failed to combine them (a *reasoning*
+deficit — decomposition scaffolds, which do nothing for missing knowledge).
+Picking the wrong one costs a whole M5 tier.
+
+The discriminating move needs no dataset annotation: ask the model to state the
+facts it needs, then hand *its own* stated facts back and ask again.  If it now
+answers correctly, the knowledge was present all along and the composition
+failed.  When the case carries gold context (``metadata['context']`` /
+``'facts'``) an open-book arm is added, which separates "does not know" from
+"cannot retrieve from its own weights".
+
+Cost: 4 generations per case, 5 with gold context.
+
+References:
+- MuSiQue: Multihop Questions via Single-hop Question Composition — Trivedi et
+  al., TACL 2022 — arXiv:2108.00573 (composability vs single-hop knowledge)
+- Measuring and Narrowing the Compositionality Gap in Language Models — Press et
+  al., EMNLP Findings 2023 — arXiv:2210.03350
+- Chain-of-Thought Prompting Elicits Reasoning — Wei et al., NeurIPS 2022 —
+  arXiv:2201.11903 (decomposition arm)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import answer_equal, extract_answer
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_ANSWER_FORMAT = "Give the final answer on its own last line as 'Answer: <answer>'."
+_BASELINE = _ANSWER_FORMAT
+_DECOMPOSE = (
+    "Break this into the sub-questions you must answer first, answer each in "
+    f"turn, then combine them. {_ANSWER_FORMAT}"
+)
+_RECALL = (
+    "Do NOT answer the question yet. List only the specific facts you would "
+    "need in order to answer it, one per line."
+)
+_FROM_FACTS = (
+    "Facts you listed:\n{facts}\n\nUsing only these facts, answer the question. "
+    f"{_ANSWER_FORMAT}"
+)
+_OPEN_BOOK = (
+    "Reference material:\n{context}\n\nUsing this material, answer the question. "
+    f"{_ANSWER_FORMAT}"
+)
+
+KNOWLEDGE = "knowledge"
+REASONING = "reasoning"
+BOTH = "both"
+NONE = "none"
+UNKNOWN = "unknown"
+
+
+@register_analyzer("knowledge_reasoning_split")
+class KnowledgeReasoningSplit(Analyzer):
+    """Attribute each failure to a missing fact or a broken composition.
+
+    Hyper-parameters:
+        use_context:  add the open-book arm when the case carries gold context.
+        context_keys: metadata keys searched for that context, in order.
+        max_cases:    label-stratified cap (4–5 generations each); 0 (the default) = every case.
+        grader:       ``callable(prediction, case) -> bool | None``.
+    """
+
+    name = "knowledge_reasoning_split"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        use_context: bool = True,
+        context_keys: tuple = ("context", "facts", "supporting_facts", "passage"),
+        max_cases: int = 0,
+        grader: Optional[Callable[[Any, "FailureCase"], Optional[bool]]] = None,
+    ) -> None:
+        super().__init__(
+            use_context=use_context,
+            context_keys=tuple(context_keys),
+            max_cases=max_cases,
+        )
+        self.grader = grader or _default_grader
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            if case.expected is None:
+                per_case.append({"sample_id": case.id, "skipped": "no gold answer"})
+                continue
+            per_case.append(self._probe_case(model, case))
+
+        scored = [c for c in per_case if "deficit_class" in c]
+        failed = [c for c in scored if c["baseline_correct"] == 0]
+
+        # UNKNOWN is a residual, not a class: with no gold context the open-book
+        # arm never runs and every failure the scaffolds did not rescue lands
+        # there. Dividing by all failures would then report a MEASURED 0.0 for
+        # knowledge_deficit_share when nothing was measured at all — the same
+        # trap arith_audit avoids by excluding its own UNKNOWN from the
+        # denominator.
+        classified = [c for c in failed if c["deficit_class"] != UNKNOWN]
+
+        def _share(cls: str) -> Optional[float]:
+            return (
+                round(sum(1 for c in classified if c["deficit_class"] == cls)
+                      / len(classified), 4)
+                if classified
+                else None
+            )
+
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(scored),
+            "n_baseline_fail": len(failed),
+            "n_classified": len(classified),
+            "n_unclassified": len(failed) - len(classified),
+            "unclassified_share": (
+                round((len(failed) - len(classified)) / len(failed), 4) if failed else None
+            ),
+            "knowledge_deficit_share": _share(KNOWLEDGE),
+            "reasoning_deficit_share": _share(REASONING),
+            "both_deficit_share": _share(BOTH),
+            "decomposition_gain": _gain(scored, "decomposed_correct"),
+            "own_facts_gain": _gain(scored, "own_facts_correct"),
+            "open_book_gain": _gain(scored, "open_book_correct"),
+            "per_case": per_case,
+            "_caveat": (
+                "The three deficit shares are over CLASSIFIED failures only — "
+                "read n_classified and unclassified_share first. Separating "
+                "'knowledge' from 'both' REQUIRES gold context in "
+                "case.metadata; without it the open-book arm never runs, every "
+                "failure the scaffolds did not rescue is unclassified, and "
+                "knowledge_deficit_share is None (not 0.0). A 0.0 here means "
+                "'measured, none were knowledge deficits'; None means 'this "
+                "batch cannot answer the question'. "
+                "The arms differ in PROMPT as well as in information, so a gain "
+                "is 'this scaffold helps', not proof of where the knowledge "
+                "lives — own_facts_correct in particular re-states the question "
+                "with a cleaner context, which alone helps some models. Read "
+                "reasoning_deficit_share against a same-length no-op scaffold "
+                "before promoting a decomposition fix. Cases where the model "
+                "lists NO usable facts fall out as own_facts_parsed=0 and are "
+                "excluded from own_facts_gain rather than counted as failures. "
+                "open_book_* exists only for cases carrying gold context; its "
+                "absence is 'not measured'. INTERVENTIONAL — re-run to confirm."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase") -> dict[str, Any]:
+        entry: dict[str, Any] = {"sample_id": case.id}
+        baseline = self.grader(model.generate(self._ask(case, _BASELINE)), case)
+        decomposed = self.grader(model.generate(self._ask(case, _DECOMPOSE)), case)
+
+        facts = str(model.generate(self._ask(case, _RECALL)))
+        fact_lines = [ln.strip() for ln in facts.splitlines() if ln.strip()]
+        entry["n_listed_facts"] = len(fact_lines)
+        entry["own_facts_parsed"] = int(bool(fact_lines))
+        own_facts = None
+        if fact_lines:
+            own_facts = self.grader(
+                model.generate(
+                    self._ask(case, _FROM_FACTS.format(facts="\n".join(fact_lines)))
+                ),
+                case,
+            )
+
+        context = self._context_for(case)
+        open_book = None
+        if context:
+            open_book = self.grader(
+                model.generate(self._ask(case, _OPEN_BOOK.format(context=context))), case
+            )
+
+        if baseline is None:
+            entry["skipped"] = "ungradable baseline"
+            return entry
+        entry["baseline_correct"] = int(baseline)
+        for key, value in (
+            ("decomposed_correct", decomposed),
+            ("own_facts_correct", own_facts),
+            ("open_book_correct", open_book),
+        ):
+            if value is not None:
+                entry[key] = int(value)
+
+        entry["deficit_class"] = _classify(baseline, own_facts, decomposed, open_book)
+        return entry
+
+    def _ask(self, case: "FailureCase", instruction: str):
+        prompt = f"{case.inputs.prompt or ''}\n\n{instruction}"
+        return dataclasses.replace(case.inputs, prompt=prompt)
+
+    def _context_for(self, case: "FailureCase") -> str:
+        if not self.use_context:
+            return ""
+        for key in self.context_keys:
+            value = case.metadata.get(key)
+            if isinstance(value, (list, tuple)):
+                value = "\n".join(str(v) for v in value)
+            if value and str(value).strip():
+                return str(value).strip()
+        return ""
+
+
+def _classify(
+    baseline: bool,
+    own_facts: Optional[bool],
+    decomposed: Optional[bool],
+    open_book: Optional[bool],
+) -> str:
+    """Which deficit does the arm pattern imply for a failing baseline?"""
+    if baseline:
+        return NONE
+    # Scaffolding its own knowledge fixed it ⇒ the facts were there, the
+    # composition was not.
+    if own_facts or decomposed:
+        return REASONING
+    # Only the gold context fixed it ⇒ the facts were genuinely missing.
+    if open_book:
+        return KNOWLEDGE
+    if open_book is False:
+        # even with the facts handed over it still fails: both halves are broken
+        return BOTH
+    return UNKNOWN
+
+
+def _gain(entries: list[dict[str, Any]], key: str) -> Optional[float]:
+    """Mean arm-minus-baseline accuracy over the cases where the arm ran."""
+    pairs = [(c["baseline_correct"], c[key]) for c in entries if key in c]
+    if not pairs:
+        return None
+    return round(sum(arm - base for base, arm in pairs) / len(pairs), 4)
+
+
+def _default_grader(prediction: Any, case: "FailureCase") -> Optional[bool]:
+    if case.expected is None:
+        return None
+    return answer_equal(extract_answer(prediction), case.expected)