evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/perturbation/perturbation_battery.py

@@ -0,0 +1,274 @@
+"""Perturbation battery — does the answer track the problem, or the template?
+
+GSM-Symbolic's finding is that accuracy on a fixed benchmark hides two opposite
+brittlenesses, and both need the *same* item perturbed rather than a new item:
+
+  * **invariance breaks** — renaming a person or appending a clause that changes
+    nothing flips the answer (the model is keying on surface form),
+  * **missing sensitivity** — changing the NUMBERS leaves the answer unmoved
+    (the model is reciting a remembered result, not computing one).
+
+Neither needs a new gold label, which is what makes the battery cheap: for the
+meaning-preserving arms the answer must not move, for the meaning-altering arm
+it must.  The GSM-NoOp arm — one true but irrelevant clause — is the single
+most diagnostic of the set and gets its own column.
+
+Cost: 1 + ``len(perturbations)`` generations per case (5 by default).
+
+References:
+- GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in
+  Large Language Models — Mirzadeh et al., ICLR 2025 — arXiv:2410.05229
+- Are NLP Models really able to Solve Simple Math Word Problems? — Patel et al.,
+  NAACL 2021 — arXiv:2103.07191 (SVAMP: sensitivity to question structure)
+- Adaptive Testing / metamorphic robustness for NLP — Ribeiro et al., ACL 2020
+  (CheckList) — arXiv:2005.04118
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import extract_answer, normalize_answer
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+#: Replacement names, chosen to be unambiguous and unlikely to appear already.
+_NAME_POOL = ("Kavi", "Rania", "Tomas", "Ingrid", "Naledi", "Yusuf")
+_NAMED_ENTITY = re.compile(r"(?<![.!?]\s)(?<!^)\b([A-Z][a-z]{2,11})\b", re.MULTILINE)
+_INTEGER = re.compile(r"(?<![\w.])(\d{1,6})(?![\w.])")
+_STOP_CAPS = frozenset(
+    {
+        "The", "This", "That", "These", "Those", "There", "How", "What", "Why",
+        "When", "Which", "Who", "If", "Answer", "Question", "Step", "Let", "In",
+        "On", "At", "For", "And", "But", "So", "Then", "Each", "Every", "All",
+        "Reply", "Give", "Think", "Note", "Total", "First", "Next", "Finally",
+    }
+)
+#: A clause that is true, on-topic, and load-bearing for nothing (GSM-NoOp).
+_NOOP_CLAUSE = (
+    " Note that last year the same quantities were recorded in a different "
+    "notebook, which has since been archived."
+)
+
+PRESERVING = "preserving"
+ALTERING = "altering"
+
+
+def rename_entities(prompt: str) -> Optional[str]:
+    """Swap capitalised mid-sentence names for unused ones (meaning-preserving)."""
+    found: list[str] = []
+    for match in _NAMED_ENTITY.finditer(prompt):
+        token = match.group(1)
+        if token not in _STOP_CAPS and token not in found:
+            found.append(token)
+    if not found:
+        return None
+    mapping = {name: _NAME_POOL[i % len(_NAME_POOL)] for i, name in enumerate(found[:3])}
+    out = prompt
+    for old, new in mapping.items():
+        out = re.sub(rf"\b{re.escape(old)}\b", new, out)
+    return out if out != prompt else None
+
+
+def append_noop_clause(prompt: str) -> Optional[str]:
+    """Append one true-but-irrelevant sentence (the GSM-NoOp arm)."""
+    lines = prompt.rstrip().splitlines()
+    if not lines:
+        return None
+    # attach to the narrative, not to a trailing instruction line
+    idx = 0 if len(lines) == 1 else max(0, len(lines) - 2)
+    lines[idx] = lines[idx].rstrip() + _NOOP_CLAUSE
+    return "\n".join(lines)
+
+
+def restate_question(prompt: str) -> Optional[str]:
+    """Prepend a neutral framing line (meaning-preserving formatting change)."""
+    return f"Read the problem carefully, then solve it.\n\n{prompt}"
+
+
+def perturb_numbers(prompt: str) -> Optional[str]:
+    """Shift every standalone integer (meaning-ALTERING: the answer must move)."""
+    seen = 0
+
+    def _bump(match: re.Match) -> str:
+        nonlocal seen
+        value = int(match.group(1))
+        seen += 1
+        # +7 avoids the 0/1 identities and keeps magnitudes plausible
+        return str(value + 7 if value >= 2 else value + 3)
+
+    out = _INTEGER.sub(_bump, prompt)
+    return out if seen and out != prompt else None
+
+
+#: ``(name, kind, fn)`` — kind decides whether a changed answer is a defect.
+DEFAULT_PERTURBATIONS: tuple = (
+    ("rename_entities", PRESERVING, rename_entities),
+    ("restate_question", PRESERVING, restate_question),
+    ("noop_clause", PRESERVING, append_noop_clause),
+    ("perturb_numbers", ALTERING, perturb_numbers),
+)
+
+
+@register_analyzer("perturbation_battery")
+class PerturbationBattery(Analyzer):
+    """Metamorphic battery: invariance breaks and missing sensitivity, per case.
+
+    Hyper-parameters:
+        perturbations: ``((name, kind, fn), ...)`` with ``kind`` in
+                       ``{"preserving", "altering"}``; ``fn(prompt) -> str | None``
+                       returns ``None`` when it does not apply to that prompt.
+        max_cases:     label-stratified cap; 0 (the default) = every case.
+        gen_kwargs:    passed to ``model.generate`` — use temperature 0, or the
+                       flip rates measure decoding noise instead of brittleness.
+        answer_fn:     ``callable(text) -> str`` answer extractor.
+    """
+
+    name = "perturbation_battery"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+    signal_docs = {
+        'baseline_answer': 'What the model answered before any perturbation.',
+        'invariance_break_rate': ('Breaks on reworded input', 'For this case: share of meaning-preserving rewordings that changed the answer.'),
+        'mean_invariance_break_rate': ('Breaks on reworded input', 'Share of meaning-PRESERVING rewordings that changed the answer. The question meant the same thing; a high rate means the model is keying on surface form rather than content.'),
+        'n_altering': 'How many meaning-changing edits this case was asked under.',
+        'n_cases': 'How many cases were re-asked under perturbed wordings.',
+        'n_memorization_suspect': ('Suspected recital', 'Cases where changing the actual content left the answer unmoved — the shape of reciting a remembered result rather than working it out.'),
+        'n_preserving': 'How many meaning-preserving rewordings this case was asked under.',
+        'n_scored': 'How many of those could be compared against a baseline answer.',
+        'noop_break_rate': ('Broken by an irrelevant clause', 'Share of cases where adding one true but irrelevant sentence flipped the answer. The single most diagnostic arm of the battery: nothing about the question changed.'),
+        'noop_clause_flipped': ('Flipped by a no-op clause', "Adding one true but irrelevant sentence changed this case's answer."),
+        'restate_question_flipped': ('Flipped by rewording', "Restating the same question in different words changed this case's answer."),
+        'sensitivity_rate': ('Responds to real changes', 'For this case: share of meaning-ALTERING edits that did change the answer. Low is bad — the model ignored a change that should matter.'),
+    }
+
+    def __init__(
+        self,
+        perturbations: tuple = DEFAULT_PERTURBATIONS,
+        max_cases: int = 0,
+        gen_kwargs: Optional[dict] = None,
+        answer_fn: Optional[Callable[[Any], str]] = None,
+    ) -> None:
+        super().__init__(max_cases=max_cases, gen_kwargs=dict(gen_kwargs or {}))
+        self.perturbations = tuple(perturbations)
+        self.answer_fn = answer_fn or extract_answer
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            per_case.append(self._probe_case(model, case))
+
+        scored = [c for c in per_case if "invariance_break_rate" in c]
+        applied = {name for c in scored for name in c.get("applied", [])}
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(scored),
+            "perturbations": [name for name, _, _ in self.perturbations],
+            "applied_perturbations": sorted(applied),
+            "gen_kwargs": dict(self.gen_kwargs),
+            "mean_invariance_break_rate": _mean(
+                [c["invariance_break_rate"] for c in scored]
+            ),
+            "noop_break_rate": _mean(
+                [c["noop_clause_flipped"] for c in scored if "noop_clause_flipped" in c]
+            ),
+            "mean_sensitivity_rate": _mean(
+                [c["sensitivity_rate"] for c in scored if c["sensitivity_rate"] is not None]
+            ),
+            "n_memorization_suspect": sum(c.get("memorization_suspect", 0) for c in scored),
+            "per_case": per_case,
+            "_caveat": (
+                "Two DIFFERENT defects share this probe and must not be summed: "
+                "invariance_break_rate should be 0 (a preserving edit moved the "
+                "answer) while sensitivity_rate should be 1 (an altering edit "
+                "must move it). sensitivity_rate == 0 with a correct baseline is "
+                "the memorisation signature — cross-check contamination_score "
+                "before calling it reasoning. Every rate is measured against the "
+                "model's OWN baseline answer, not the gold, so a case that was "
+                "already wrong still contributes: read these as stability, never "
+                "as accuracy. Under sampling, a 'flip' can be decoding noise — "
+                "set temperature 0 or repeat. INTERVENTIONAL: re-run to confirm."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase") -> dict[str, Any]:
+        prompt = str(case.inputs.prompt or "")
+        entry: dict[str, Any] = {"sample_id": case.id}
+        if not prompt.strip():
+            entry["skipped"] = "empty prompt"
+            return entry
+
+        # Treat a transient backend failure as missing evidence for this case.
+        # Letting it escape aborts the whole battery, which makes the most
+        # relevant M1 evidence disappear after just one timed-out audio clip.
+        try:
+            baseline = normalize_answer(
+                self.answer_fn(model.generate(case.inputs, **self.gen_kwargs))
+            )
+        except Exception as exc:
+            entry["skipped"] = f"baseline request failed: {type(exc).__name__}: {str(exc)[:160]}"
+            return entry
+        entry["baseline_answer"] = baseline[:80]
+
+        applied: list[str] = []
+        preserving_flips: list[int] = []
+        altering_flips: list[int] = []
+        for name, kind, fn in self.perturbations:
+            try:
+                variant = fn(prompt)
+            except Exception:  # pragma: no cover - user-supplied perturbation
+                variant = None
+            if not variant or variant == prompt:
+                continue
+            applied.append(name)
+            try:
+                answer = normalize_answer(
+                    self.answer_fn(
+                        model.generate(
+                            dataclasses.replace(case.inputs, prompt=variant), **self.gen_kwargs
+                        )
+                    )
+                )
+            except Exception as exc:
+                entry.setdefault("request_errors", {})[name] = (
+                    f"{type(exc).__name__}: {str(exc)[:160]}"
+                )
+                continue
+            flipped = int(answer != baseline)
+            entry[f"{name}_flipped"] = flipped
+            (preserving_flips if kind == PRESERVING else altering_flips).append(flipped)
+
+        entry["applied"] = applied
+        if not applied:
+            entry["skipped"] = "no perturbation applies to this prompt"
+            return entry
+
+        entry["invariance_break_rate"] = (
+            round(sum(preserving_flips) / len(preserving_flips), 4)
+            if preserving_flips
+            else 0.0
+        )
+        entry["n_preserving"] = len(preserving_flips)
+        entry["sensitivity_rate"] = (
+            round(sum(altering_flips) / len(altering_flips), 4) if altering_flips else None
+        )
+        entry["n_altering"] = len(altering_flips)
+        if altering_flips and sum(altering_flips) == 0:
+            # the numbers changed and the answer did not: recall, not computation
+            entry["memorization_suspect"] = 1
+        return entry
+
+
+def _mean(values: list) -> Optional[float]:
+    clean = [v for v in values if v is not None]
+    return round(sum(clean) / len(clean), 4) if clean else None

evalrx/analyzers/perturbation/prompt_contrast.py

@@ -0,0 +1,265 @@
+"""Prompt-contrast — intervention-grade evidence via paired prompt strategies.
+
+Black-box (``GENERATE``): re-asks every case under k prompt strategies (the
+unchanged baseline + targeted rewrites) and scores each answer against the
+case's ``expected`` rubric.  The output feeds the paired statistics machinery:
+
+- ``findings["by_strategy"] = {strategy: {case_id: 0/1}}`` → M2's
+  ``mcnemar_evalue`` (paired, anytime-valid) and ``friedman_nemenyi`` tools;
+- per-case repair flags (``fixed_by_<strategy>``) → ``signal_label_assoc``.
+
+This is the harness's *causal* probe for decision-layer failure hypotheses:
+if "describe the region first, then answer" repairs failures, the deficit is in
+answer generation, not perception ("language prior bias"); if an explicit
+"report even subtle findings" instruction repairs them, the model's internal
+yes-threshold is miscalibrated; if nothing helps, the failure sits in visual
+feature discrimination itself.
+
+The default strategies are designed to discriminate exactly those mechanisms;
+pass ``strategies={name: template}`` for custom interventions (templates use
+``{prompt}`` for the original question).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+# Default interventions — chosen to discriminate decision-layer mechanisms.
+_DEFAULT_STRATEGIES: dict[str, str] = {
+    "baseline": "{prompt}",
+    "describe_first": (
+        "First describe what you see in the relevant region of the image in "
+        "one sentence, then answer the question. {prompt}"
+    ),
+    "sensitive": (
+        "{prompt} Report a finding as present even if it is subtle — do not "
+        "default to a negative answer when uncertain."
+    ),
+}
+
+
+_YESNO_RE = re.compile(r"\b(yes|no)\b")
+
+
+def _norm_answer(text: str) -> str:
+    """Coarse, task-agnostic answer key for cross-prompt agreement.
+
+    Yes/no answers (POPE-style) collapse to ``yes``/``no``; otherwise the
+    lowercased, whitespace-collapsed first 48 chars stand in. Good enough to tell
+    "the model said the same thing under every prompt" from "its answer flipped"
+    — a correctness-INDEPENDENT measure, so it never tracks the FAIL label."""
+    t = re.sub(r"\s+", " ", str(text).strip().lower())
+    m = _YESNO_RE.search(t)
+    return m.group(1) if m else t[:48]
+
+
+def _prompt_sensitivity(answers: dict, cid: str, strategies) -> "float | None":
+    """Disagreement rate among the strategies' answers for one case.
+
+    0.0 = every prompt yielded the same answer (prompt-robust); → 1.0 = the
+    answer flips across prompts (prompt-fragile). Independent of whether any
+    answer is correct, so it enters M2 as a genuine signal, not a label copy:
+    a confident hallucination is robust (≈0), a borderline case is fragile."""
+    from collections import Counter
+
+    vals = [_norm_answer(answers[s][cid]) for s in strategies if cid in answers.get(s, {})]
+    if len(vals) < 2:
+        return None
+    return round(1.0 - Counter(vals).most_common(1)[0][1] / len(vals), 4)
+
+
+def _word_in(term: str, text: str) -> bool:
+    """Word-boundary match for plain alphanumeric terms, substring otherwise."""
+    term = term.lower().strip()
+    if not term:
+        return False
+    if re.fullmatch(r"[a-z0-9]+", term):
+        return re.search(rf"\b{re.escape(term)}\b", text) is not None
+    return term in text
+
+
+def _default_score(case: "FailureCase", observed: str) -> Optional[bool]:
+    """Score *observed* against ``case.expected`` (dict rubric or string).
+
+    Returns ``True``/``False``, or ``None`` when the case has no usable rubric.
+    """
+    expected = getattr(case, "expected", None)
+    if expected is None:
+        return None
+    if isinstance(expected, (list, tuple, set, frozenset)):
+        from evalrx.analyzers.reasoning._text import default_grader
+
+        return default_grader(observed, case)
+    text = re.sub(r"\s+", " ", str(observed).lower())
+    if isinstance(expected, dict):
+        if any(_word_in(t, text) for t in expected.get("none_of", [])):
+            return False
+        if not all(_word_in(t, text) for t in expected.get("all_of", [])):
+            return False
+        any_of = expected.get("any_of", [])
+        if any_of and not any(_word_in(t, text) for t in any_of):
+            return False
+        return True
+    if isinstance(expected, str):
+        return _word_in(expected, text)
+    return None
+
+
+@register_analyzer("prompt_contrast")
+class PromptContrastAnalyzer(Analyzer):
+    """Paired prompt-strategy contrast over the whole batch.
+
+    Hyper-parameters:
+        strategies: ``{name: template}`` prompt rewrites; templates receive the
+                    original question via ``{prompt}``.  Must include a
+                    ``"baseline"`` entry (added automatically if missing).
+        score_fn:   ``(case, answer) -> bool | None`` answer scorer.  Defaults
+                    to a word-boundary rubric scorer over ``case.expected``
+                    (dict ``all_of/any_of/none_of`` or plain string).
+        max_cases:  Cap on cases (cost = ``len(strategies)`` generations each); 0 (the default) = every case.
+    """
+
+    name = "prompt_contrast"
+    description = (
+        "Re-asks each question under paired prompt interventions (baseline / "
+        "describe-the-image-first / answer-sensitively) and scores each variant "
+        "— causal evidence for whether failures are repairable at the prompt "
+        "level (language prior / threshold miscalibration) or persist "
+        "regardless (perception failure)."
+    )
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+    signal_docs = {
+        'n_broken_by_describe_first': ('Broken by describe-first', 'Cases that were right and became wrong. A rewrite that fixes some and breaks others is not an improvement.'),
+        'n_broken_by_sensitive': ('Broken by report-subtle', 'Cases that were right and became wrong under the same instruction.'),
+        'n_cases': 'How many cases were re-asked under each prompt strategy.',
+        'n_fixed_by_describe_first': ('Fixed by describe-first', 'Cases that were wrong and became right when told to describe first.'),
+        'n_fixed_by_sensitive': ('Fixed by report-subtle', 'Cases that were wrong and became right when told to report subtle findings.'),
+        'n_strategies': 'How many prompt strategies were compared, including the unchanged baseline.',
+        'n_unscored': 'Answers that could not be graded, so they enter no comparison.',
+        'prompt_sensitivity': ('Answer moves with wording', "How much this case's answer changed across the prompt rewrites. 0 means every wording produced the same answer; high means the answer depends on how the question was phrased. Independent of whether any answer was correct."),
+        'success_rate_baseline': ('Correct, unchanged prompt', 'Share correct with the original question — the line every rewrite is measured against.'),
+        'success_rate_describe_first': ('Correct, describe-first', 'Share correct when told to describe what it perceives before answering. Higher than baseline means the failure was in answering, not in perceiving.'),
+        'success_rate_sensitive': ('Correct, report-subtle', "Share correct when told to report a finding even if subtle. Higher than baseline means the model's yes/no threshold was set too high, not that it could not tell."),
+    }
+
+    def __init__(
+        self,
+        strategies: dict[str, str] | None = None,
+        score_fn: Callable[[Any, str], Optional[bool]] | None = None,
+        # Sized for enriched batches: the per-strategy McNemar is powered by the
+        # scarce FAIL group (a strategy can only *flip* an already-failing case),
+        # so a small cap leaves too few discordant pairs for M4 to clear the
+        # e-value bar. 128 covers typical mined fail counts plus controls.
+        max_cases: int = 0,
+    ) -> None:
+        strategies = dict(strategies) if strategies else dict(_DEFAULT_STRATEGIES)
+        if "baseline" not in strategies:
+            strategies = {"baseline": "{prompt}", **strategies}
+        super().__init__(strategies=strategies, score_fn=score_fn, max_cases=max_cases)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        score = self.score_fn or _default_score
+        by_strategy: dict[str, dict[str, float]] = {s: {} for s in self.strategies}
+        answers: dict[str, dict[str, str]] = {s: {} for s in self.strategies}
+        n_unscored = 0
+
+        # Stratified cap: keep FAIL representation so the cases actually carry
+        # flippable failures (a plain head is mostly PASS on an enriched batch,
+        # or all-FAIL on a fail-first ordering — both lopsided). stratified_head
+        # mixes both, which the McNemar (flips) and the no-free-lunch check
+        # (breaks of correct answers) each need.
+        selected = cases.stratified_head(self.max_cases)
+        for case in selected:
+            prompt = str(getattr(case.inputs, "prompt", ""))
+            for strat, template in self.strategies.items():
+                rewritten = template.format(prompt=prompt)
+                # dataclasses.replace, not a bare Inputs(prompt=..., image=...):
+                # this analyzer is also compatible with audio models (it
+                # declares "text" in applies_to_modalities, which intersects
+                # any modality set that includes text prompts) — a fresh
+                # Inputs(image=...) silently dropped .audio, so the re-ask
+                # would have answered without hearing the clip.
+                out = str(model.generate(dataclasses.replace(case.inputs, prompt=rewritten)))
+                verdict = score(case, out)
+                answers[strat][case.id] = out[:200]
+                if verdict is None:
+                    n_unscored += 1
+                    continue
+                by_strategy[strat][case.id] = float(bool(verdict))
+
+        # Per-case repair/breakage flags relative to baseline — stored in
+        # ARTIFACTS, not findings["per_case"].  As label-association signals
+        # they are tautological: fixed_by_* can only fire on baseline-FAIL
+        # cases and broken_by_* only on baseline-PASS cases, so their sign
+        # against the labels is structurally determined and produces spurious
+        # SUPPORTED/REFUTED verdicts (same family as the removed
+        # baseline_correct flag).  The rigorous intervention verdict lives in
+        # the paired McNemar over by_strategy; the per-case flags remain
+        # available in artifacts for inspection.
+        base = by_strategy.get("baseline", {})
+        variants = [s for s in self.strategies if s != "baseline"]
+        per_case: list[dict[str, Any]] = []
+        for case in selected:
+            cid = case.id
+            if cid not in base:
+                continue
+            entry: dict[str, Any] = {"sample_id": cid}
+            for strat in variants:
+                if cid in by_strategy[strat]:
+                    entry[f"fixed_by_{strat}"] = bool(
+                        base[cid] == 0.0 and by_strategy[strat][cid] == 1.0
+                    )
+                    entry[f"broken_by_{strat}"] = bool(
+                        base[cid] == 1.0 and by_strategy[strat][cid] == 0.0
+                    )
+            per_case.append(entry)
+
+        # Per-case signal surfaced to M2 (findings["per_case"]): prompt
+        # sensitivity — how much the model's ANSWER flips across the prompt
+        # strategies, independent of correctness. Unlike the fixed_by_/broken_by_
+        # flags above (tautological, kept in artifacts), this does NOT track the
+        # FAIL label, so it can enter signal_label_assoc / rank_corr cleanly.
+        findings_per_case: list[dict[str, Any]] = []
+        for case in selected:
+            sens = _prompt_sensitivity(answers, case.id, self.strategies)
+            if sens is not None:
+                findings_per_case.append({"sample_id": case.id, "prompt_sensitivity": sens})
+
+        findings: dict[str, Any] = {
+            "n_cases": len(selected),
+            "n_strategies": len(self.strategies),
+            "n_unscored": n_unscored,
+            "by_strategy": by_strategy,
+            "per_case": findings_per_case,
+        }
+        for strat in self.strategies:
+            scored = by_strategy[strat]
+            if scored:
+                findings[f"success_rate_{strat}"] = round(
+                    sum(scored.values()) / len(scored), 4
+                )
+        for strat in variants:
+            n_fixed = sum(1 for e in per_case if e.get(f"fixed_by_{strat}"))
+            n_broken = sum(1 for e in per_case if e.get(f"broken_by_{strat}"))
+            findings[f"n_fixed_by_{strat}"] = n_fixed
+            findings[f"n_broken_by_{strat}"] = n_broken
+
+        return Result(
+            analyzer=self.name,
+            model=repr(model),
+            cases=cases,
+            findings=findings,
+            artifacts={"answers": answers, "per_case": per_case},
+        )

evalrx/analyzers/perturbation/rise.py

@@ -0,0 +1,94 @@
+"""RISE — Randomized Input Sampling for Explanation (Petsiuk et al.), text variant.
+
+Black-box, model-agnostic: mask random subsets of input tokens, score each masked
+output, and attribute importance to a token as the mean score when it is kept.
+``requires=GENERATE`` plus an injected ``score_fn(output)->float`` (correctness /
+verifier signal — task-specific, so the caller provides it).
+
+Cost note: this is the dominant compute (n_masks forward passes per sample). For
+API models, run it through the existing async caller with an on-disk cache keyed
+by the mask hash so a crashed sweep resumes.
+
+References:
+- RISE: Randomized Input Sampling for Explanation of Black-box Models
+  Petsiuk, Das & Saenko, BMVC 2018 — arXiv:1806.07421
+"""
+
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING, Callable, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("rise")
+class RISEAnalyzer(Analyzer):
+    """Token-occlusion saliency over the input prompt.
+
+    Hyper-parameters:
+        score_fn:  ``output_text -> float`` scalar (required).
+        n_masks:   number of random masks (cost driver).
+        keep_prob: probability a token is kept in a mask.
+        mask_token: replacement for dropped tokens.
+        top_k:     tokens to report.
+        seed:      RNG seed (reproducibility).
+    """
+
+    name = "rise"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text"})
+
+    def __init__(
+        self,
+        score_fn: Optional[Callable[[str], float]] = None,
+        n_masks: int = 50,
+        keep_prob: float = 0.5,
+        mask_token: str = "___",
+        top_k: int = 5,
+        seed: int = 0,
+    ) -> None:
+        super().__init__(
+            score_fn=score_fn, n_masks=n_masks, keep_prob=keep_prob,
+            mask_token=mask_token, top_k=top_k, seed=seed,
+        )
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        if self.score_fn is None:
+            raise ValueError("RISEAnalyzer requires score_fn(output)->float (a task-specific verifier).")
+        rng = random.Random(self.seed)
+        case = cases[0]
+        words = str(case.inputs).split()
+        n = len(words)
+        sal = [0.0] * n
+        cnt = [0] * n
+        n_calls = 0
+        for _ in range(self.n_masks):
+            mask = [rng.random() < self.keep_prob for _ in range(n)]
+            if not any(mask):
+                continue
+            masked = " ".join(w if m else self.mask_token for w, m in zip(words, mask))
+            score = float(self.score_fn(model.generate(masked)))
+            n_calls += 1
+            for j, keep in enumerate(mask):
+                if keep:
+                    sal[j] += score
+                    cnt[j] += 1
+        importance = [(sal[j] / cnt[j]) if cnt[j] else 0.0 for j in range(n)]
+        ranked = sorted(zip(words, importance), key=lambda x: -x[1])[: self.top_k]
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"importance": importance, "words": words},
+            findings={
+                "n_words": n,
+                "n_masks_run": n_calls,
+                "top_tokens": [{"token": w, "importance": round(v, 4)} for w, v in ranked],
+            },
+        )