evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/hallucination/opera.py

@@ -0,0 +1,29 @@
+"""OPERA — over-trust penalty + retrospection-allocation on attention (Stage 2).
+
+Diagnoses object hallucination from the attention pattern (over-trust on a few
+summary tokens). ``requires=ATTENTION`` (+ decode-loop control for the mitigation).
+White-box, VLM.
+
+References:
+- OPERA: Alleviating Hallucination in MLLMs via Over-trust Penalty and Retrospection-Allocation
+  Huang et al., CVPR 2024 — arXiv:2311.17911
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+
+
+@register_analyzer("opera")
+class OPERAAnalyzer(Analyzer):
+    name = "opera"
+    requires = frozenset({Capability.ATTENTION})
+    applies_to_modalities = frozenset({"image"})
+    # An omni model DECLARES image even on an audio benchmark, so the registry
+    # match alone still offers this on a batch with no images in it.
+    requires_modalities = frozenset({"image"})
+
+    def _run(self, model, cases):
+        raise NotImplementedError("Stage 2: attention over-trust diagnosis (needs decode-loop control).")

evalrx/analyzers/hallucination/pope.py

@@ -0,0 +1,119 @@
+"""POPE — Polling-based Object Probing Evaluation.
+
+Black-box (``GENERATE``): pose yes/no object-presence questions about an image and
+score the model's answers against gold labels.  Each case carries the gold label
+in ``metadata["pope_label"]`` ("yes"/"no") — build a probe set with the datasets
+layer.  Reports accuracy / precision / recall / F1 (positive class = "yes").
+
+Paper: "Evaluating Object Hallucination in Large Vision-Language Models"
+       Li et al., EMNLP 2023 — https://arxiv.org/abs/2305.10355
+Code:  https://github.com/AoiDragon/POPE
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+def parse_yes_no(text: str) -> Optional[str]:
+    """Map a free-form answer to 'yes' / 'no' / None (word-boundary, not substring).
+
+    Matching whole words avoids the classic trap where 'not'/'none'/'nothing'
+    contain the substring 'no'.
+    """
+    words = re.findall(r"[a-z]+", str(text).lower())
+    for w in words[:6]:  # decision is almost always in the opening words
+        if w == "yes":
+            return "yes"
+        if w == "no":
+            return "no"
+    if "yes" in words:
+        return "yes"
+    if "no" in words:
+        return "no"
+    return None
+
+
+@register_analyzer("pope")
+class POPEAnalyzer(Analyzer):
+    """Score yes/no object-presence probes against gold labels."""
+
+    name = "pope"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"image"})
+    # An omni model DECLARES image even on an audio benchmark, so the registry
+    # match alone still offers this on a batch with no images in it.
+    requires_modalities = frozenset({"image"})
+
+    def __init__(self, label_key: str = "pope_label") -> None:
+        super().__init__(label_key=label_key)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        tp = fp = fn = tn = unparsed = 0
+        per_case = []
+        for case in cases:
+            gold = str(case.metadata.get(self.label_key, "")).strip().lower()
+            pred = parse_yes_no(model.generate(case.inputs))
+            is_correct = pred == gold if gold in {"yes", "no"} and pred is not None else None
+            per_case.append({
+                "id": case.id,
+                "gold": gold,
+                "pred": pred,
+                # Mechanism signals first: M2's stats layer caps how many per-case
+                # signal keys it tests, and these two carry the diagnosis
+                # (hallucinated presence vs missed finding) — is_correct alone
+                # cannot distinguish them.
+                "false_positive": gold == "no" and pred == "yes",
+                "false_negative": gold == "yes" and pred == "no",
+                # The two MARGINALS those flags are built from. false_positive /
+                # false_negative are gold x pred conjunctions -- each a subset of
+                # FAIL by construction, so the planner keeps them out of the
+                # tested family (descriptive only). answered_yes / gold_yes are
+                # not label functions and are what M2/M4 test a directional
+                # ("over-affirmation") hypothesis on. Same columns as
+                # answer_extraction_audit emits on any yes/no task.
+                **({"answered_yes": pred == "yes"} if pred is not None else {}),
+                **({"gold_yes": gold == "yes"} if gold in {"yes", "no"} else {}),
+                "has_gold": gold in {"yes", "no"},
+                "unparsed": pred is None,
+                "is_correct": is_correct,
+            })
+            if pred is None:
+                unparsed += 1
+                continue
+            if gold == "yes" and pred == "yes":
+                tp += 1
+            elif gold == "no" and pred == "yes":
+                fp += 1
+            elif gold == "yes" and pred == "no":
+                fn += 1
+            elif gold == "no" and pred == "no":
+                tn += 1
+        n = tp + fp + fn + tn
+        precision = tp / (tp + fp) if (tp + fp) else 0.0
+        recall = tp / (tp + fn) if (tp + fn) else 0.0
+        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"per_case": per_case},
+            findings={
+                "n": n, "unparsed": unparsed,
+                "accuracy": round((tp + tn) / n, 4) if n else None,
+                "precision": round(precision, 4),
+                "recall": round(recall, 4),
+                "f1": round(f1, 4),
+                "yes_rate": round((tp + fp) / n, 4) if n else None,  # >0.5 ⇒ over-affirmation (hallucination)
+                "false_positive_rate": round(fp / (fp + tn), 4) if (fp + tn) else None,
+                "per_case": per_case,
+            },
+        )

evalrx/analyzers/hallucination/selfcheck.py

@@ -0,0 +1,155 @@
+"""Sampled-passage factual consistency — the text counterpart of POPE/CHAIR (black-box).
+
+SelfCheckGPT's premise: if the model KNOWS a fact, independently sampled
+generations agree on it; hallucinated content diverges across samples.  This
+port implements the dependency-free unigram-containment variant: each sentence
+of the baseline answer is scored by how well the resampled passages cover its
+content words.  ``requires=GENERATE`` only, so it runs on API models.
+
+The score is a per-case M2 column, not a truth oracle: correlate
+``selfcheck_inconsistency`` with the outcome label downstream instead of
+treating a high score as a verdict.
+
+References:
+- SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative
+  Large Language Models — Manakul et al., EMNLP 2023 — arXiv:2303.08896
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+_WORD = re.compile(r"[a-z0-9]+")
+_SENT_SPLIT = re.compile(r"(?<=[.!?])\s+|(?<=[\u3002\uff01\uff1f])\s*|\n+")
+
+
+def split_sentences(text: str, min_chars: int = 10) -> list[str]:
+    """Split into scoreable sentences, dropping fragments below ``min_chars``."""
+    parts = [p.strip() for p in _SENT_SPLIT.split(str(text or ""))]
+    return [p for p in parts if len(p) >= min_chars]
+
+
+def _char_bigrams(text: str) -> set[str]:
+    chars = [c for c in text if not c.isspace()]
+    return {a + b for a, b in zip(chars, chars[1:])} or set(chars)
+
+
+def containment(sentence: str, passage: str) -> float:
+    """Fraction of the sentence's content units that appear in the passage.
+
+    Latin/digit words when the sentence has any; character bigrams otherwise,
+    so CJK (or symbol-only) sentences are scored instead of silently passing.
+    """
+    sent = set(_WORD.findall(sentence.lower()))
+    if sent:
+        passage_words = set(_WORD.findall(passage.lower()))
+        return len(sent & passage_words) / len(sent)
+    bigrams = _char_bigrams(sentence.lower())
+    if not bigrams:
+        return 1.0
+    return len(bigrams & _char_bigrams(passage.lower())) / len(bigrams)
+
+
+@register_analyzer("selfcheck_consistency")
+class SelfCheckConsistencyAnalyzer(Analyzer):
+    """SelfCheckGPT-style sampled-passage consistency: sentence-level hallucination signal from resampling.
+
+    Hyper-parameters:
+        n_samples:  resampled passages per case (baseline answer excluded).
+        gen_kwargs: sampling config for the resamples — MUST be stochastic
+                    (default ``{"temperature": 1.0}``); at temperature 0 the
+                    score is degenerate and reported as such.
+        max_cases:  label-stratified cap on probed cases; 0 (the default) = every case.
+        min_sentence_chars: shorter fragments are not scored.
+    """
+
+    name = "selfcheck_consistency"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        n_samples: int = 4,
+        gen_kwargs: Optional[dict] = None,
+        max_cases: int = 0,
+        min_sentence_chars: int = 10,
+    ) -> None:
+        super().__init__(
+            n_samples=max(1, int(n_samples)),
+            gen_kwargs=dict(gen_kwargs) if gen_kwargs is not None else {"temperature": 1.0},
+            max_cases=max_cases,
+            min_sentence_chars=min_sentence_chars,
+        )
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            baseline = str(case.observed or "").strip()
+            baseline_from = "observed"
+            if not baseline:
+                baseline = str(model.generate(case.inputs))
+                baseline_from = "generated"
+            sentences = split_sentences(baseline, self.min_sentence_chars)
+            entry: dict[str, Any] = {
+                "sample_id": case.id,
+                "baseline_from": baseline_from,
+                "n_sentences": len(sentences),
+            }
+            if not sentences:
+                entry["skipped"] = "no scoreable sentence in the baseline answer"
+                per_case.append(entry)
+                continue
+            samples = [
+                str(model.generate(case.inputs, **self.gen_kwargs))
+                for _ in range(self.n_samples)
+            ]
+            # SelfCheckGPT scores a sentence by its support in OTHER samples;
+            # a sentence no resample reproduces is the hallucination suspect.
+            sentence_scores = [
+                1.0 - max(containment(sent, sample) for sample in samples)
+                for sent in sentences
+            ]
+            entry["selfcheck_inconsistency"] = round(
+                sum(sentence_scores) / len(sentence_scores), 4
+            )
+            entry["selfcheck_worst_sentence"] = round(max(sentence_scores), 4)
+            worst = sentences[sentence_scores.index(max(sentence_scores))]
+            entry["worst_sentence_text"] = worst[:200]
+            per_case.append(entry)
+
+        scored = [c["selfcheck_inconsistency"] for c in per_case if "selfcheck_inconsistency" in c]
+        temperature = self.gen_kwargs.get("temperature")
+        degenerate = (
+            (temperature is not None and float(temperature) == 0.0)
+            or self.gen_kwargs.get("do_sample") is False
+        )
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_samples": self.n_samples,
+            "degenerate_sampling": degenerate,
+            "gen_kwargs": dict(self.gen_kwargs),
+            "mean_inconsistency": round(sum(scored) / len(scored), 4) if scored else None,
+            "per_case": per_case,
+            "_caveat": (
+                "Unigram-containment proxy for SelfCheckGPT (no NLI model): "
+                "paraphrases inflate the score, verbatim repetition of a wrong "
+                "fact deflates it. Requires stochastic sampling — with "
+                "temperature 0 the resamples repeat one greedy generation and the "
+                "column degenerates to baseline-vs-greedy disagreement "
+                "(degenerate_sampling flags this). "
+                "selfcheck_inconsistency / selfcheck_worst_sentence are numeric "
+                "M2 columns; treat them as signals to correlate with the outcome "
+                "label, never as a hallucination verdict."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)

evalrx/analyzers/hallucination/vcd.py

@@ -0,0 +1,29 @@
+"""VCD — Visual Contrastive Decoding (Leng et al.) (Stage 2).
+
+Contrast logits on the original vs a distorted image to suppress
+language-prior-driven hallucination. Needs decode-time logit control (in-process
+HF), VLM. Modeled here as an analyzer that reports the contrastive shift.
+
+References:
+- Mitigating Object Hallucinations in LVLMs through Visual Contrastive Decoding (VCD)
+  Leng et al., CVPR 2024 — arXiv:2311.16922
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+
+
+@register_analyzer("vcd")
+class VCDAnalyzer(Analyzer):
+    name = "vcd"
+    requires = frozenset({Capability.LOGITS})
+    applies_to_modalities = frozenset({"image"})
+    # An omni model DECLARES image even on an audio benchmark, so the registry
+    # match alone still offers this on a batch with no images in it.
+    requires_modalities = frozenset({"image"})
+
+    def _run(self, model, cases):
+        raise NotImplementedError("Stage 2: contrastive decoding over original vs distorted image (logit control).")

evalrx/analyzers/lens/__init__.py

@@ -0,0 +1,7 @@
+"""Lens analyzers — project hidden states toward the vocabulary."""
+
+from evalrx.analyzers.lens.layer_contrast import LayerContrastAnalyzer
+from evalrx.analyzers.lens.logit_lens import LogitLensAnalyzer
+from evalrx.analyzers.lens.tuned_lens import TunedLensAnalyzer
+
+__all__ = ["LogitLensAnalyzer", "TunedLensAnalyzer", "LayerContrastAnalyzer"]

evalrx/analyzers/lens/layer_contrast.py

@@ -0,0 +1,133 @@
+"""Premature-layer vs final-layer divergence — the DoLa/DeCo family's diagnostic half.
+
+DoLa's observation: factual knowledge surfaces in specific layers, and tokens
+whose final distribution diverges sharply from premature layers are where
+"what the layers know" and "what gets decoded" disagree — a hallucination
+indicator.  DeCo's follow-up: hallucinated targets are often *correct* in
+preceding layers and suppressed at the top.  This analyzer measures those
+dynamics per case WITHOUT changing decoding: it emits divergence/agreement
+columns that, when they separate FAIL from PASS in M2, point the fix search at
+layer-contrast decoding repairs (DoLa/DeCo/VISTA family).
+
+White-box: ``requires=HIDDEN_STATES`` plus the model's unembedding
+(``unembed_weight()``), same access contract as :class:`LogitLensAnalyzer`.
+
+References:
+- DoLa: Decoding by Contrasting Layers Improves Factuality in Large Language
+  Models — Chuang et al., ICLR 2024 — arXiv:2309.03883
+- MLLM Can See? Dynamic Correction Decoding for Hallucination Mitigation
+  (DeCo) — Wang et al., ICLR 2025 — arXiv:2410.11779
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("layer_contrast")
+class LayerContrastAnalyzer(Analyzer):
+    """DoLa-style premature/final layer divergence columns (JSD, agreement, contrast margin).
+
+    Hyper-parameters:
+        pos:             query position to read (default ``-1``, the last token).
+        skip_first_frac: fraction of early layers excluded from the premature
+                         candidate set (embedding-adjacent layers are noise).
+        max_cases:       label-stratified cap (one forward per case); 0 (the default) = every case.
+    """
+
+    name = "layer_contrast"
+    requires = frozenset({Capability.HIDDEN_STATES})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(self, pos: int = -1, skip_first_frac: float = 0.25, max_cases: int = 0) -> None:
+        super().__init__(pos=pos, skip_first_frac=float(skip_first_frac), max_cases=max_cases)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        import torch
+
+        W = model.unembed_weight()
+        if W is None:
+            raise ValueError(
+                f"{type(model).__name__} exposes no unembed_weight(); layer-contrast "
+                "needs it (white-box local backend)."
+            )
+        W = W.detach().float()
+        device = W.device
+        norm = model.final_norm() if hasattr(model, "final_norm") else None
+        norm_dtype = next(norm.parameters()).dtype if norm is not None and any(
+            True for _ in norm.parameters()) else None
+
+        def _jsd(p: "torch.Tensor", q: "torch.Tensor") -> float:
+            m = 0.5 * (p + q)
+            kl_pm = torch.sum(p * (torch.log(p + 1e-12) - torch.log(m + 1e-12)))
+            kl_qm = torch.sum(q * (torch.log(q + 1e-12) - torch.log(m + 1e-12)))
+            return float(0.5 * (kl_pm + kl_qm))
+
+        per_case: list[dict[str, Any]] = []
+        n_layers = 0
+        for case in cases.stratified_head(self.max_cases):
+            trace = model.forward(case.inputs, capture={Capability.HIDDEN_STATES})
+            hidden = trace.require(Capability.HIDDEN_STATES)  # list per layer: (seq, dim)
+            n_layers = len(hidden)
+            with torch.no_grad():
+                probs = []
+                for index, h in enumerate(hidden):
+                    vec = h[self.pos].detach().to(device)
+                    # HF backends return the FINAL hidden-states entry already
+                    # post final-norm; re-normalising it would square the RMS
+                    # gain and distort every divergence below.
+                    if norm is not None and index < len(hidden) - 1:
+                        vec = norm(vec.to(norm_dtype)) if norm_dtype is not None else norm(vec)
+                    probs.append(torch.softmax(vec.float() @ W.T, dim=-1))
+                final = probs[-1]
+                top1 = int(final.argmax())
+                start = min(int(self.skip_first_frac * n_layers), max(0, n_layers - 2))
+                candidates = list(range(start, n_layers - 1))
+                if not candidates:
+                    candidates = [max(0, n_layers - 2)]
+                jsds = [_jsd(final, probs[layer]) for layer in candidates]
+                jsd_max = max(jsds)
+                max_layer = candidates[jsds.index(jsd_max)]
+                agree = sum(1 for layer in candidates if int(probs[layer].argmax()) == top1)
+                # DoLa contrast margin at the max-divergence premature layer:
+                # positive = the final layer amplifies its top-1 over the
+                # premature layer; negative = the top-1 was STRONGER early and
+                # got suppressed at the top (DeCo's hallucination signature).
+                contrast_margin = float(
+                    torch.log(final[top1] + 1e-12) - torch.log(probs[max_layer][top1] + 1e-12)
+                )
+            per_case.append({
+                "sample_id": case.id,
+                "jsd_max": round(jsd_max, 4),
+                "jsd_mean": round(sum(jsds) / len(jsds), 4),
+                "jsd_max_layer_frac": round(max_layer / max(1, n_layers - 1), 4),
+                "layer_agreement_frac": round(agree / len(candidates), 4),
+                "contrast_margin": round(contrast_margin, 4),
+                "final_top1_prob": round(float(final[top1]), 4),
+            })
+
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_layers": n_layers,
+            "pos": self.pos,
+            "premature_layer_window": f"[{self.skip_first_frac:.2f}L, L-1)",
+            "per_case": per_case,
+            "_caveat": (
+                "Divergence DIAGNOSTICS only — no decoding intervention is "
+                "performed. If jsd_max / negative contrast_margin separate FAIL "
+                "from PASS in M2, the indicated repair family is layer-contrast "
+                "decoding (DoLa/DeCo/VISTA), an L0/L3 fix. Columns read one "
+                "position (pos) of a teacher-forced forward; generation-time "
+                "dynamics can differ."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)

evalrx/analyzers/lens/logit_lens.py

@@ -0,0 +1,138 @@
+"""Logit lens (nostalgebraist) — project intermediate hidden states through the unembed.
+
+Reveals *when* (which layer) the model's prediction forms.  Needs ``HIDDEN_STATES``
+plus the model's unembedding matrix (``model.unembed_weight()``).  The cheapest,
+highest-value white-box signal — build it first.
+
+Faithfulness: when the model exposes ``final_norm()`` (RMSNorm-family models —
+Llama, Qwen, ...), each PREMATURE layer's hidden state is normalized before
+unembedding, matching ``lm_head(norm(h_i))``; raw projection distorts
+trajectories on those models.  The final hidden-states entry is projected
+as-is: HF backends return it already post final-norm, so re-normalising it
+would square the RMS gain and distort the reference distribution.  Findings
+note whether the norm was applied.
+
+References:
+- interpreting GPT: the logit lens — nostalgebraist (2020), LessWrong
+  https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("logit_lens")
+class LogitLensAnalyzer(Analyzer):
+    """Per-layer readout at one position, plus per-case decision-depth signals.
+
+    Per case (``findings["per_case"]``, consumable by M2's stats layer):
+        decision_layer / decision_frac: first layer whose top-1 equals the
+            final top-1 — late decisions mean the answer formed in the last
+            layers (prior-override candidates).
+        final_top1_prob:  confidence of the emitted token at the final layer.
+        interim_peak_prob / late_drop: peak probability of the final top-1
+            across layers and its drop to the final layer (non-monotonic
+            trajectories flag suppression dynamics).
+
+    Hyper-parameters:
+        pos:       query position to read (default ``-1``, the last token).
+        top_k:     number of candidates to report per layer (first case only).
+        max_cases: cap on cases analyzed (one forward each); 0 (the default) = every case.
+    """
+
+    name = "logit_lens"
+    requires = frozenset({Capability.HIDDEN_STATES})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    # Default cap sized for ENRICHED diagnosis batches: the per-layer fail/pass
+    # contrast is powered by the scarce FAIL group, and a small cap (e.g. 32)
+    # starves it — on a curated batch the stratified subsample then carries only
+    # ~16 FAILs, too few for M4 to clear the e-value bar. 128 covers typical
+    # mined fail counts (tens) plus controls at one forward each.
+    def __init__(self, pos: int = -1, top_k: int = 3, max_cases: int = 0) -> None:
+        super().__init__(pos=pos, top_k=top_k, max_cases=max_cases)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        import torch
+
+        W = model.unembed_weight()
+        if W is None:
+            raise ValueError(
+                f"{type(model).__name__} exposes no unembed_weight(); logit-lens needs it "
+                "(white-box local backend)."
+            )
+        W = W.detach().float()  # (vocab, dim), once — not per layer
+        device = W.device
+        norm = model.final_norm() if hasattr(model, "final_norm") else None
+        norm_dtype = next(norm.parameters()).dtype if norm is not None and any(
+            True for _ in norm.parameters()) else None
+
+        # Label-stratified subsample: a plain head is mostly PASS on an enriched
+        # batch, starving the FAIL group that the downstream contrast needs.
+        per_case, per_layer_top, n_layers = [], None, 0
+        for case in cases.stratified_head(self.max_cases):
+            trace = model.forward(case.inputs, capture={Capability.HIDDEN_STATES})
+            hidden = trace.require(Capability.HIDDEN_STATES)  # list per layer: (seq, dim)
+            n_layers = len(hidden)
+
+            layer_logits = []
+            with torch.no_grad():
+                for layer_index, h in enumerate(hidden):
+                    vec = h[self.pos].detach().to(device)  # device-align: states may be on CPU
+                    # HF backends return the FINAL hidden-states entry already
+                    # post final-norm (this analyzer requests the full stack, so
+                    # the last element IS that entry); re-normalising it would
+                    # square the RMS gain and shift the final distribution the
+                    # decision-depth columns are anchored to.
+                    if norm is not None and layer_index < len(hidden) - 1:
+                        vec = norm(vec.to(norm_dtype)) if norm_dtype is not None else norm(vec)
+                    layer_logits.append(vec.float() @ W.T)  # (vocab,)
+
+                final = layer_logits[-1]
+                top1 = int(final.argmax())
+                p_top1 = [float(torch.softmax(lg, dim=-1)[top1]) for lg in layer_logits]
+                argmaxes = [int(lg.argmax()) for lg in layer_logits]
+            decision_layer = next(
+                (i for i in range(n_layers) if all(a == top1 for a in argmaxes[i:])), n_layers - 1)
+            peak = max(p_top1)
+            per_case.append({
+                "sample_id": case.id,
+                "decision_layer": decision_layer,
+                "decision_frac": round(decision_layer / max(1, n_layers - 1), 4),
+                "final_top1_prob": round(p_top1[-1], 4),
+                "interim_peak_prob": round(peak, 4),
+                "late_drop": round(peak - p_top1[-1], 4),
+            })
+
+            if per_layer_top is None:  # detailed per-layer table for the first case only
+                per_layer_top = []
+                for layer_idx, lg in enumerate(layer_logits):
+                    k = min(self.top_k, lg.shape[-1])
+                    topk = torch.topk(lg, k)
+                    per_layer_top.append({
+                        "layer": layer_idx,
+                        "top": [{"token_id": int(i), "logit": round(float(v), 3)}
+                                for v, i in zip(topk.values, topk.indices)],
+                    })
+
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"per_case": per_case},
+            findings={
+                "n_layers": n_layers, "pos": self.pos,
+                "n_cases_analyzed": len(per_case),
+                "final_norm_applied": norm is not None,
+                "per_layer_top": per_layer_top or [],
+                "per_case": per_case,
+            },
+        )

evalrx/analyzers/lens/tuned_lens.py

@@ -0,0 +1,30 @@
+"""Tuned lens (Belrose et al.) — per-layer learned affine translators before unembed.
+
+More faithful than the raw logit lens, but requires PER-MODEL trained translators.
+Stage 2: train/load translators, then apply like the logit lens.
+
+References:
+- Eliciting Latent Predictions from Transformers with the Tuned Lens
+  Belrose et al., 2023 — arXiv:2303.08112
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+
+
+@register_analyzer("tuned_lens")
+class TunedLensAnalyzer(Analyzer):
+    """Tuned-lens projection of hidden states (needs trained translators)."""
+
+    name = "tuned_lens"
+    requires = frozenset({Capability.HIDDEN_STATES})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def _run(self, model, cases):
+        raise NotImplementedError(
+            "Stage 2: load/train per-layer affine translators (e.g. via the `tuned-lens` package), "
+            "then project hidden states through them before the unembed."
+        )

evalrx/analyzers/patching/__init__.py

@@ -0,0 +1,5 @@
+"""Causal intervention analyzers (activation patching / causal tracing)."""
+
+from evalrx.analyzers.patching.causal_trace import CausalTraceAnalyzer
+
+__all__ = ["CausalTraceAnalyzer"]