evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/perturbation/format_sensitivity.py

@@ -0,0 +1,237 @@
+"""Multiple-choice format sensitivity — does the answer follow content or position?
+
+LLMs (and VLMs answering MC benchmarks) carry measurable selection bias: some
+models keep the same LETTER when options are reordered, instead of following
+the option CONTENT.  This probe rotates the option list, re-asks, and reports
+per-case columns separating the two failure signatures:
+
+    format_flip_rate  — answers track neither content nor a fixed position
+                        (brittle / guessing);
+    positional_bias   — answers keep choosing the same letter across rotations
+                        (position prior overrides perception).
+
+Black-box (``requires=GENERATE``); rotations are deterministic, so held-out
+verification can re-run the probe exactly.
+
+References:
+- Large Language Models Are Not Robust Multiple Choice Selectors —
+  Zheng et al., ICLR 2024 — arXiv:2309.03882
+- Quantifying Language Models' Sensitivity to Spurious Features in Prompt
+  Design (FormatSpread) — Sclar et al., ICLR 2024 — arXiv:2310.11324
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from collections import Counter
+from typing import TYPE_CHECKING, Any, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+#: Matches ``A. opt``, ``A) opt`` and ``(A) opt``.
+#:
+#: The leading ``\(?`` is not cosmetic. BBH, AGIEval and several MMLU
+#: redistributions write ``(A) blue ball``, and without it the letter is not at
+#: the start of the line, so the whole block matched nothing and the analyzer
+#: reported ``n_scored=0`` on a purely multiple-choice benchmark — a silent
+#: no-op that still spent a generation budget getting there.
+_OPTION_LINE = re.compile(r"^[ \t]*\(?([A-Z])[.)][ \t]+(.+?)[ \t]*$", re.MULTILINE)
+_PAREN_OPTION = re.compile(r"^[ \t]*\([A-Z]\)")
+_MAX_LETTERS = 26
+
+
+def extract_option_block(prompt: str) -> tuple[list[str], Optional[str]]:
+    """The LAST contiguous lettered run ``A. ... B. ...`` in the prompt.
+
+    A run must start at ``A``, increment by one letter per line, and contain
+    only whitespace between consecutive option lines — this keeps few-shot
+    examples and restated blocks from being merged into one bogus list.
+    Returns ``(options, exact_block_text)`` or ``([], None)``.
+    """
+    text = str(prompt or "")
+    runs: list[list[re.Match]] = []
+    current: list[re.Match] = []
+    for match in _OPTION_LINE.finditer(text):
+        letter = match.group(1)
+        if letter == "A":
+            current = [match]
+        elif (
+            current
+            and ord(letter) == ord(current[-1].group(1)) + 1
+            and not text[current[-1].end():match.start()].strip()
+        ):
+            current.append(match)
+        else:
+            current = []
+            continue
+        if len(current) >= 2:
+            runs.append(list(current))
+    if not runs:
+        return [], None
+    best = runs[-1]  # final state of the last run
+    options = [m.group(2) for m in best]
+    return options, text[best[0].start():best[-1].end()]
+
+
+def extract_options(case: "FailureCase") -> tuple[list[str], Optional[str]]:
+    """Options from ``metadata['options']`` (no block text) or the prompt's lettered block."""
+    meta = case.metadata.get("options") if isinstance(case.metadata, dict) else None
+    if isinstance(meta, (list, tuple)) and 2 <= len(meta) <= _MAX_LETTERS:
+        return [str(o) for o in meta], None
+    options, block = extract_option_block(case.inputs.prompt or "")
+    if not 2 <= len(options) <= _MAX_LETTERS:
+        return [], None
+    return options, block
+
+
+def render_options(options: list[str], paren: bool = False) -> str:
+    """Render a rotated block, keeping the prompt's own letter style.
+
+    ``paren`` matters for correctness, not looks: this probe isolates POSITIONAL
+    bias by rotating options. Re-rendering a ``(A)`` block as ``A.`` would change
+    the delimiter at the same time as the position, so any flip it measured
+    could be either cause.
+    """
+    if paren:
+        return "\n".join(f"({chr(65 + i)}) {opt}" for i, opt in enumerate(options))
+    return "\n".join(f"{chr(65 + i)}. {opt}" for i, opt in enumerate(options))
+
+
+def parse_choice(output: str, n_options: int) -> str:
+    """Chosen letter: a tagged answer first, else the last standalone CAPITAL.
+
+    The tag pattern is case-insensitive on the raw text; the fallback scans the
+    raw (non-uppercased) output so the article 'a' cannot masquerade as option
+    A, and a capital immediately followed by a lowercase word (\"A nice ...\")
+    is treated as prose, not a choice.
+    """
+    n = min(int(n_options), _MAX_LETTERS)
+    if n < 1:
+        return ""
+    last = chr(64 + n)  # 'A' + n - 1
+    text = str(output or "")
+    marked = re.findall(
+        # \(? so a model that answers in the prompt's own style -- "Answer: (A)"
+        # on a "(A) ..." block -- is read as a tagged answer rather than falling
+        # through to the positional fallback below
+        rf"(?:answer|choice|option|final)\s*(?:is)?\s*[:=-]?\s*\(?([A-{last}])\b",
+        text,
+        re.IGNORECASE,
+    )
+    if marked:
+        return marked[-1].upper()
+    letters = re.findall(rf"\b([A-{last}])\b(?!\s+[a-z])", text)
+    return letters[-1] if letters else ""
+
+
+@register_analyzer("format_sensitivity")
+class FormatSensitivityAnalyzer(Analyzer):
+    """Option-rotation probe: separates positional letter bias from content-tracking on multiple-choice cases.
+
+    Hyper-parameters:
+        n_variants: rotations asked per case (rotation k shifts options by k;
+                    capped at n_options - 1).
+        max_cases:  label-stratified cap on probed cases; 0 (the default) = every case.
+    """
+
+    name = "format_sensitivity"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(self, n_variants: int = 3, max_cases: int = 0) -> None:
+        super().__init__(n_variants=max(1, int(n_variants)), max_cases=max_cases)
+
+    @staticmethod
+    def _variant_prompt(
+        prompt: str, block: Optional[str], options: list[str], shift: int
+    ) -> tuple[str, list[str]]:
+        rotated = options[shift:] + options[:shift]
+        paren = bool(block) and bool(_PAREN_OPTION.match(block))
+        rotated_block = render_options(rotated, paren=paren)
+        if block:
+            # identical restatements of the same block rotate together
+            return prompt.replace(block, rotated_block), rotated
+        return (
+            f"{prompt}\n\nChoices:\n{rotated_block}\nAnswer with the option letter.",
+            rotated,
+        )
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            options, block = extract_options(case)
+            entry: dict[str, Any] = {"sample_id": case.id, "n_options": len(options)}
+            if len(options) < 2:
+                entry["skipped"] = "no multiple-choice options found (2..26 required)"
+                per_case.append(entry)
+                continue
+            prompt = case.inputs.prompt or ""
+            shifts = list(range(1, min(self.n_variants, len(options) - 1) + 1))
+            chosen_contents: list[str] = []
+            chosen_letters: list[str] = []
+            n_unparsed = 0
+            # Identity variant: reuse the recorded baseline answer only when the
+            # prompt contains the lettered block the letters decode against;
+            # otherwise every variant (identity included) uses OUR rendered
+            # block, so all letters map to options the model actually saw.
+            if block is not None and case.observed:
+                base_out = str(case.observed)
+            else:
+                identity_prompt, _ = self._variant_prompt(prompt, block, options, 0)
+                base_out = str(
+                    model.generate(dataclasses.replace(case.inputs, prompt=identity_prompt))
+                )
+            base_letter = parse_choice(base_out, len(options))
+            if base_letter:
+                chosen_letters.append(base_letter)
+                chosen_contents.append(options[ord(base_letter) - 65])
+            else:
+                n_unparsed += 1
+            for shift in shifts:
+                variant_prompt, rotated = self._variant_prompt(prompt, block, options, shift)
+                out = str(model.generate(dataclasses.replace(case.inputs, prompt=variant_prompt)))
+                letter = parse_choice(out, len(rotated))
+                if not letter:
+                    n_unparsed += 1
+                    continue
+                chosen_letters.append(letter)
+                chosen_contents.append(rotated[ord(letter) - 65])
+            entry["n_variants"] = 1 + len(shifts)
+            entry["n_unparsed"] = n_unparsed
+            if chosen_contents:
+                content_modal = Counter(chosen_contents).most_common(1)[0][1]
+                letter_modal = Counter(chosen_letters).most_common(1)[0]
+                entry["format_flip_rate"] = round(1.0 - content_modal / len(chosen_contents), 4)
+                entry["positional_bias"] = round(letter_modal[1] / len(chosen_letters), 4)
+                entry["modal_letter"] = letter_modal[0]
+            per_case.append(entry)
+
+        flips = [c["format_flip_rate"] for c in per_case if "format_flip_rate" in c]
+        letters = Counter(c.get("modal_letter") for c in per_case if c.get("modal_letter"))
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(flips),
+            "mean_flip_rate": round(sum(flips) / len(flips), 4) if flips else None,
+            "modal_letter_histogram": dict(letters),
+            "per_case": per_case,
+            "_caveat": (
+                "Rotations only (not all permutations): a model biased toward "
+                "'the option after the correct one' can evade detection. "
+                "positional_bias near 1 with format_flip_rate near 1 is the "
+                "letter-prior signature; both near 0 is content-tracking. "
+                "Letter parsing prefers tagged answers and ignores capitals "
+                "followed by a lowercase word, but free-prose answers can still "
+                "be misread — n_unparsed tracks the abstentions. INTERVENTIONAL "
+                "columns: held-out verification must RE-RUN the rotations, "
+                "never reuse exploration-set values."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)

evalrx/analyzers/perturbation/mm_shap.py

@@ -0,0 +1,146 @@
+"""MM-SHAP — modality-contribution metric, generalised to every media slot.
+
+Black-box (``LOGPROBS``): Shapley over the input's text tokens plus one player
+per FILLED media slot (image / audio / video), scored by the model's output
+logprob, then aggregated by modality::
+
+    <slot>_contribution / (text_contribution + Σ media contributions)
+
+0 for a slot ⇒ the prediction ignores it; 1 ⇒ it carries the prediction alone.
+Measures *reliance*, not correctness — report it as such.
+
+The original metric had exactly one non-text player, the image, and this
+implementation hard-coded that: the value function rebuilt ``Inputs(prompt=...,
+image=...)`` and dropped audio and video on the floor.  On an audio benchmark it
+therefore reported ``mm_score`` computed over a slot that was never filled — the
+question "is this model actually listening" was unanswerable by the one analyzer
+whose whole job is answering it.  Players are now the slots the case fills, so
+an AVLM gets a per-slot breakdown and ``mm_score`` keeps its published meaning
+(image reliance) as one member of it.
+
+Paper: "MM-SHAP: A Performance-agnostic Metric for Measuring Multimodal
+       Contributions in Vision and Language Models"
+       Parcalabescu & Frank, ACL 2022 — https://arxiv.org/abs/2212.08158
+Code:  https://github.com/coastalcph/mm-shap
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, Optional
+
+from evalrx.analyzers.perturbation._shapley import shapley_values
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.case import MEDIA_SLOTS, Inputs
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+#: Player key for a media slot. Namespaced so it can never collide with a word
+#: index, which is a plain int.
+def _slot_player(slot: str) -> str:
+    return f"__{slot}__"
+
+
+#: Kept for backwards compatibility with callers that imported the old constant.
+_IMAGE = _slot_player("image")
+
+
+@register_analyzer("mm_shap")
+class MMShapAnalyzer(Analyzer):
+    """Per-modality Shapley contribution + the MM-SHAP media-reliance scores."""
+
+    name = "mm_shap"
+    requires = frozenset({Capability.LOGPROBS})
+    applies_to_modalities = frozenset({"text", "image", "audio", "video"})
+    signal_docs = {
+        'audio_contribution': 'How much the audio drove the answer.',
+        'audio_score': ("Audio's share", "The audio's share of everything that drove the answer."),
+        'has_audio': 'Whether this case had audio at all.',
+        'has_image': 'Whether this case had an image at all.',
+        'has_video': 'Whether this case had video at all.',
+        'image_contribution': 'How much the image drove the answer.',
+        'image_score': ("Image's share", "The image's share of everything that drove the answer."),
+        'media_score': ('Reliance on media', 'How much the answer rested on the pictures and sound together, rather than on the question text alone.'),
+        'mm_score': ('Reliance on the image', 'How much the answer rested on the image, from 0 (all text) to 1 (all image).'),
+        'probed_slots': 'Which modalities were present to weigh.',
+        'text_contribution': ('Reliance on the question', "How much the question's own wording drove the answer."),
+        'video_contribution': 'How much the video drove the answer.',
+        'video_score': ("Video's share", "The video's share of everything that drove the answer."),
+    }
+    #: Needs at least one media slot filled — with none, every player is a word
+    #: and the "modality contribution" it reports is a text-only tautology.
+    requires_modalities = frozenset(MEDIA_SLOTS)
+
+    def __init__(
+        self,
+        score_fn: Optional[Callable[[Inputs], float]] = None,
+        n_samples: int = 64,
+        mask_token: str = "___",
+        top_k: int = 5,
+        seed: int = 0,
+    ) -> None:
+        super().__init__(score_fn=score_fn, n_samples=n_samples, mask_token=mask_token, top_k=top_k, seed=seed)
+
+    def _default_scorer(self, model: "Model") -> Callable[[Inputs], float]:
+        def score(inputs: Inputs) -> float:
+            lps = model.logprobs(inputs)
+            return sum(t.logprob for t in lps) / len(lps) if lps else 0.0
+        return score
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        case = cases[0]
+        words = str(case.inputs.prompt).split()
+        filled = [s for s in MEDIA_SLOTS if getattr(case.inputs, s, None) is not None]
+        score = self.score_fn or self._default_scorer(model)
+
+        players: list = list(range(len(words)))
+        players += [_slot_player(s) for s in filled]
+
+        def value(kept: set) -> float:
+            masked = " ".join(w if i in kept else self.mask_token for i, w in enumerate(words))
+            slots = {
+                s: (getattr(case.inputs, s) if _slot_player(s) in kept else None)
+                for s in filled
+            }
+            return score(Inputs(prompt=masked, **slots))
+
+        shap = shapley_values(players, value, n_samples=self.n_samples, seed=self.seed)
+        text_contrib = sum(abs(shap[i]) for i in range(len(words)))
+        media_contrib = {s: abs(shap.get(_slot_player(s), 0.0)) for s in filled}
+        total = text_contrib + sum(media_contrib.values()) or 1.0
+        top_text = sorted(
+            ({"token": words[i], "shapley": round(shap[i], 4)} for i in range(len(words))),
+            key=lambda d: -abs(d["shapley"]),
+        )[: self.top_k]
+
+        findings = {
+            # Published MM-SHAP: image reliance. 0.0 when no image is present,
+            # which is why the per-slot fields below carry the real answer for an
+            # audio or video run rather than this one.
+            "mm_score": round(media_contrib.get("image", 0.0) / total, 4),
+            "media_score": round(sum(media_contrib.values()) / total, 4),
+            "text_contribution": round(text_contrib, 4),
+            "probed_slots": filled,
+            "top_text_tokens": top_text,
+            "_note": "measures modality reliance, not correctness",
+        }
+        for slot in MEDIA_SLOTS:
+            # Every slot gets a key, present or not: a reader must be able to tell
+            # "measured as zero" from "never in this case", and an absent key
+            # collapses the two into whatever their default lookup returns.
+            findings[f"has_{slot}"] = slot in filled
+            findings[f"{slot}_contribution"] = (
+                round(media_contrib[slot], 4) if slot in filled else None
+            )
+            findings[f"{slot}_score"] = (
+                round(media_contrib[slot] / total, 4) if slot in filled else None
+            )
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"shapley": shap},
+            findings=findings,
+        )

evalrx/analyzers/perturbation/modality_ablation.py

@@ -0,0 +1,196 @@
+"""Modality ablation — is the model actually using the modality under test?
+
+Black-box (``GENERATE``): for every media slot a case fills, re-ask the same
+question with that slot **ablated**, and compare the answer to the unablated
+baseline.  An answer that does not move when the audio is removed or replaced
+was not produced by listening to it.
+
+One implementation covers every slot, and that is the point.  The question "is
+the image load-bearing" and the question "is the audio load-bearing" are the
+same question asked of a different slot, so a per-modality analyzer would be the
+same code three times, drifting apart — and in practice the audio and video
+copies were simply never written, leaving an ALM run with no way to ask it at
+all.  Adding a modality here is adding a member to
+:data:`~evalrx.core.case.MEDIA_SLOTS`.
+
+Two ablation modes, because they answer different questions:
+
+* ``drop`` (default) — remove the slot entirely.  An unchanged answer is strong
+  evidence the slot was never read.  A *changed* answer proves only sensitivity,
+  not correct use: the input shape changed too.
+* ``swap`` — substitute another case's media, keeping the input shape intact and
+  breaking only the grounding.  Cleaner evidence, and it needs no extra model
+  capability, but it requires at least two cases filling the same slot.
+
+Reported as reliance, never as correctness.  ``ungrounded_rate_<slot>`` is the
+share of probed cases whose answer survived the ablation; ``by_strategy`` carries
+the paired per-case correctness that M2's McNemar reads, which is what makes this
+INTERVENTION-grade rather than one more association.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.perturbation.prompt_contrast import _default_score, _norm_answer
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.case import MEDIA_SLOTS
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+BASELINE = "baseline"
+
+
+@register_analyzer("modality_ablation")
+class ModalityAblationAnalyzer(Analyzer):
+    """Per-slot grounding probe: does ablating a modality move the answer?"""
+
+    name = "modality_ablation"
+    description = (
+        "Re-asks each case with one media slot removed or swapped; an unchanged "
+        "answer means the model never used that modality."
+    )
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image", "audio", "video"})
+    signal_docs = {
+        'ablation_audio_is_swap': "The audio was replaced with another case's rather than removed.",
+        'ablation_image_is_swap': "The image was replaced with another case's rather than removed.",
+        'ablation_mode_audio': 'How the audio was removed for these cases.',
+        'ablation_mode_image': 'How the image was removed for these cases.',
+        'ablation_mode_video': 'How the video was removed for these cases.',
+        'ablation_video_is_swap': "The video was replaced with another case's rather than removed.",
+        'grounded_in_audio': ('Actually listened', 'The answer changed when the audio was taken away — the model was listening.'),
+        'grounded_in_image': ('Actually looked', 'The answer changed when the image was taken away — the model was using it.'),
+        'grounded_in_video': ('Actually watched', 'The answer changed when the video was taken away — the model was watching.'),
+        'mode': "How the modality was removed: dropped entirely, or swapped for another case's.",
+        'n_cases_probed': 'How many cases were re-asked with a modality removed.',
+        'n_probed_audio': 'How many cases had audio to test.',
+        'n_probed_image': 'How many cases had an image to test.',
+        'n_probed_video': 'How many cases had video to test.',
+        'probed_slots': 'Which modalities were tested this way.',
+        'ungrounded_rate_audio': ('Ignored the audio', 'Share of cases whose answer did NOT change when the audio was removed — the model was not listening.'),
+        'ungrounded_rate_image': ('Ignored the image', 'Share of cases whose answer did NOT change when the image was removed — the model was not looking.'),
+        'ungrounded_rate_video': ('Ignored the video', 'Share of cases whose answer did NOT change when the video was removed — the model was not watching.'),
+    }
+    #: Needs a filled slot to ablate. Without one there is nothing to remove and
+    #: the analyzer would report a grounding rate over zero probes.
+    requires_modalities = frozenset(MEDIA_SLOTS)
+
+    def __init__(
+        self,
+        mode: str = "drop",
+        slots: "tuple[str, ...] | None" = None,
+        score_fn: Optional[Callable[["FailureCase", str], Optional[bool]]] = None,
+        max_cases: int = 0,
+    ) -> None:
+        """
+        Args:
+            mode:      ``"drop"`` (remove the slot) or ``"swap"`` (substitute
+                       another case's media). ``swap`` falls back to ``drop``
+                       per-slot when fewer than two cases fill it.
+            slots:     Restrict to these slots; ``None`` probes every filled one.
+            score_fn:  ``(case, answer) -> bool | None`` for the paired
+                       correctness table. Defaults to the prompt-contrast rubric
+                       scorer; ``None`` verdicts are simply not tabulated.
+            max_cases: Cap on cases (cost = 1 + n_filled_slots generations each);
+                       0 (the default) = every case.
+        """
+        if mode not in ("drop", "swap"):
+            raise ValueError(f"mode must be 'drop' or 'swap'; got {mode!r}")
+        super().__init__(mode=mode, slots=slots, score_fn=score_fn, max_cases=max_cases)
+
+    # -- helpers -------------------------------------------------------
+    def _slots_of(self, case: "FailureCase") -> "list[str]":
+        wanted = self.slots or MEDIA_SLOTS
+        return [
+            s for s in MEDIA_SLOTS
+            if s in wanted and getattr(case.inputs, s, None) is not None
+        ]
+
+    @staticmethod
+    def _donor(cases: "list[FailureCase]", slot: str, case: "FailureCase") -> Any:
+        """Another case's media for *slot*, or ``None`` when there is no donor.
+
+        Deterministic (first non-self filler in batch order) so a rerun of the
+        same batch ablates with the same substitute — a random donor would make
+        the paired comparison irreproducible.
+        """
+        for other in cases:
+            if other.id != case.id and getattr(other.inputs, slot, None) is not None:
+                return getattr(other.inputs, slot)
+        return None
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        score = self.score_fn or _default_score
+        selected = cases.stratified_head(self.max_cases)
+        selected = [c for c in selected if self._slots_of(c)]
+
+        by_strategy: dict[str, dict[str, float]] = {BASELINE: {}}
+        per_case: list[dict[str, Any]] = []
+        # per-slot tallies: probed cases, and those whose answer never moved
+        probed = {s: 0 for s in MEDIA_SLOTS}
+        ungrounded = {s: 0 for s in MEDIA_SLOTS}
+        modes_used: dict[str, str] = {}
+
+        for case in selected:
+            base_answer = str(model.generate(case.inputs))
+            base_key = _norm_answer(base_answer)
+            base_verdict = score(case, base_answer)
+            if base_verdict is not None:
+                by_strategy[BASELINE][case.id] = float(bool(base_verdict))
+
+            row: dict[str, Any] = {"sample_id": case.id}
+            for slot in self._slots_of(case):
+                donor = self._donor(selected, slot, case) if self.mode == "swap" else None
+                # swap with no donor degrades to drop rather than skipping: the
+                # weaker evidence is still evidence, and silently probing nothing
+                # would look identical to a fully-grounded model.
+                effective = "swap" if donor is not None else "drop"
+                modes_used[slot] = effective
+                ablated = dataclasses.replace(case.inputs, **{slot: donor})
+
+                answer = str(model.generate(ablated))
+                changed = _norm_answer(answer) != base_key
+
+                probed[slot] += 1
+                if not changed:
+                    ungrounded[slot] += 1
+                # Flat scalars only: the per-case harvester reads one level, so a
+                # nested {slot: {...}} here would reach no statistic at all.
+                row[f"grounded_in_{slot}"] = bool(changed)
+                row[f"ablation_{slot}_is_swap"] = effective == "swap"
+
+                strategy = f"without_{slot}"
+                verdict = score(case, answer)
+                if verdict is not None:
+                    by_strategy.setdefault(strategy, {})[case.id] = float(bool(verdict))
+            if len(row) > 1:
+                per_case.append(row)
+
+        findings: dict[str, Any] = {
+            "n_cases_probed": len(selected),
+            "mode": self.mode,
+            "probed_slots": sorted(s for s in MEDIA_SLOTS if probed[s]),
+            "_note": "measures whether a modality is load-bearing, not correctness",
+            "per_case": per_case,
+            "by_strategy": by_strategy,
+        }
+        for slot in MEDIA_SLOTS:
+            n = probed[slot]
+            # None, not 0.0: "no case filled this slot" and "every case that
+            # filled it stayed grounded" are different facts and a reader that
+            # cannot separate them will report the wrong one.
+            findings[f"n_probed_{slot}"] = n
+            findings[f"ungrounded_rate_{slot}"] = round(ungrounded[slot] / n, 4) if n else None
+            findings[f"ablation_mode_{slot}"] = modes_used.get(slot)
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            findings=findings,
+            metadata={"mode": self.mode, "n_selected": len(selected)},
+        )