evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/attention/sink.py

@@ -0,0 +1,56 @@
+"""Attention sink (Gu et al., "When Attention Sink Emerges in LMs").
+
+Measures how much attention mass collapses onto the first token (the "sink"),
+per layer, averaged over heads and query positions.  Operates on a ``Trace``.
+
+References:
+- Efficient Streaming Language Models with Attention Sinks (StreamingLLM)
+  Xiao et al., ICLR 2024 — arXiv:2309.17453
+- When Attention Sink Emerges in Language Models: An Empirical View
+  Gu et al., ICLR 2025 — arXiv:2410.10781
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("attention_sink")
+class AttentionSinkAnalyzer(Analyzer):
+    """Per-layer attention mass on the first token (sink strength)."""
+
+    name = "attention_sink"
+    requires = frozenset({Capability.ATTENTION})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(self, sink_pos: int = 0) -> None:
+        super().__init__(sink_pos=sink_pos)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        case = cases[0]
+        trace = model.forward(case.inputs, capture={Capability.ATTENTION})
+        attns = trace.require(Capability.ATTENTION)
+        per_layer = []
+        for a in attns:
+            A = a.float().mean(0)                  # (seq, seq), head-averaged
+            per_layer.append(round(float(A[:, self.sink_pos].mean()), 4))
+        mean_sink = round(sum(per_layer) / max(len(per_layer), 1), 4)
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"per_layer_sink": per_layer},
+            findings={
+                "n_layers": len(attns),
+                "sink_token": trace.tokens[self.sink_pos] if trace.tokens else None,
+                "mean_sink_mass": mean_sink,
+                "per_layer_sink": per_layer,
+            },
+        )

evalrx/analyzers/attention/summary.py

@@ -0,0 +1,190 @@
+"""Attention analysis for white-box models.
+
+``AttentionAnalyzer`` is a :class:`~evalrx.core.analyzer.Analyzer`: configure
+it with hyper-parameters, then ``run`` it on any model that provides the
+``ATTENTION`` capability.  It returns an :class:`AttentionResult`, whose
+``findings`` are a light, agent-readable summary and whose ``artifacts`` hold the
+raw per-layer/per-head tensors for plotting and numeric work.
+
+References:
+- "What's in the Image?" — vision attention deep-dive
+- "VL-Cache" — modality-aware KV-cache
+- "When Attention Sink Emerges in Language Models" — https://arxiv.org/abs/2309.17453
+- "MLLMs Know Where to Look" (relative attention) — https://arxiv.org/abs/2502.17422
+  · code: https://github.com/saccharomycetes/mllms_know
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import torch
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+# ======================================================================
+# Result
+# ======================================================================
+
+@dataclass
+class AttentionResult(Result):
+    """Attention analysis output for a single case.
+
+    The heavy per-layer attention tensors live in ``artifacts["attentions"]``
+    (each ``(heads, seq, seq)``).  ``findings`` carries the agent-readable
+    summary.  The convenience methods below operate on the artifacts.
+    """
+
+    tokens: list[str] = field(default_factory=list)
+    token_ids: list[int] = field(default_factory=list)
+
+    # -- artifact accessors --------------------------------------------
+    @property
+    def attentions(self) -> list[torch.Tensor]:
+        return self.artifacts["attentions"]
+
+    @property
+    def num_layers(self) -> int:
+        return len(self.attentions)
+
+    @property
+    def num_heads(self) -> int:
+        return self.attentions[0].shape[0]
+
+    @property
+    def seq_len(self) -> int:
+        return len(self.tokens)
+
+    # -- core aggregation ----------------------------------------------
+    def aggregate(self, layer: int = -1, head: int | str = "mean") -> torch.Tensor:
+        """Return a ``(seq_len, seq_len)`` attention matrix.
+
+        Args:
+            layer: Layer index (negative indexing supported; ``-1`` = last).
+            head:  Head index, or ``"mean"`` to average over heads.
+        """
+        attn = self.attentions[layer]
+        return attn.mean(dim=0) if head == "mean" else attn[int(head)]
+
+    def layer_head_matrix(self) -> torch.Tensor:
+        """Stacked tensor of shape ``(layers, heads, seq_len, seq_len)``."""
+        return torch.stack(self.attentions, dim=0)
+
+    def to_numpy(self, layer: int = -1, head: int | str = "mean") -> np.ndarray:
+        return self.aggregate(layer=layer, head=head).cpu().float().numpy()
+
+    # -- inspection ----------------------------------------------------
+    def top_attended_tokens(
+        self,
+        query_pos: int = -1,
+        k: int = 10,
+        layer: int = -1,
+        head: int | str = "mean",
+    ) -> list[tuple[str, float]]:
+        """Top-k tokens attended to by the token at *query_pos*."""
+        row = self.aggregate(layer=layer, head=head)[query_pos]
+        k = min(k, self.seq_len)
+        topk = torch.topk(row, k)
+        return [(self.tokens[i], float(v)) for v, i in zip(topk.values, topk.indices)]
+
+    def attention_entropy(self, layer: int = -1, head: int | str = "mean") -> torch.Tensor:
+        """Shannon entropy of the attention distribution per query position."""
+        mat = self.aggregate(layer=layer, head=head).float().clamp(min=1e-9)
+        return -(mat * mat.log()).sum(dim=-1)
+
+    # -- visualisation -------------------------------------------------
+    def plot(self, layer: int = -1, head: int | str = "mean", ax=None, figsize=(8, 7)):
+        """Plot the attention matrix as a heatmap (requires ``evalrx[viz]``)."""
+        try:
+            import matplotlib.pyplot as plt
+        except ImportError:
+            raise ImportError(
+                "Visualisation requires matplotlib. Install: pip install evalrx[viz]"
+            ) from None
+
+        created = ax is None
+        if created:
+            fig, ax = plt.subplots(figsize=figsize)
+        mat = self.to_numpy(layer=layer, head=head)
+        im = ax.imshow(mat, cmap="viridis", aspect="auto", vmin=0)
+        ax.set_xticks(range(self.seq_len))
+        ax.set_xticklabels(self.tokens, rotation=90, fontsize=7)
+        ax.set_yticks(range(self.seq_len))
+        ax.set_yticklabels(self.tokens, fontsize=7)
+        ax.set_title(f"Attention | layer={layer} head={head}")
+        plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
+        if created:
+            plt.tight_layout()
+            return fig
+        return ax
+
+
+# ======================================================================
+# Analyzer
+# ======================================================================
+
+@register_analyzer("attention")
+class AttentionAnalyzer(Analyzer):
+    """Extract and summarise attention weights from a white-box model.
+
+    Hyper-parameters (sklearn-style, passed to ``__init__``):
+        layer: Default layer for the ``findings`` summary (``-1`` = last).
+        head:  Default head for the summary (``"mean"`` or an int).
+        top_k: Number of top attended tokens to include in ``findings``.
+
+    Example::
+
+        result = AttentionAnalyzer(layer=-1, top_k=5).run(qwen, "The capital of France is")
+        result.summary()
+        result.plot()
+    """
+
+    name = "attention"
+    requires = frozenset({Capability.ATTENTION})
+
+    def __init__(self, layer: int = -1, head: int | str = "mean", top_k: int = 10) -> None:
+        super().__init__(layer=layer, head=head, top_k=top_k)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> AttentionResult:
+        # Stage 1: analyse the first case (single-case ergonomics). Multi-case
+        # aggregation is a Stage-2 extension that will return a batched result.
+        case = cases[0]
+        trace = model.forward(case.inputs, capture={Capability.ATTENTION})
+        attentions = trace.require(Capability.ATTENTION)
+
+        result = AttentionResult(
+            analyzer=self.name,
+            model=repr(model),
+            cases=cases,
+            tokens=trace.tokens,
+            token_ids=trace.token_ids,
+            artifacts={"attentions": attentions},
+        )
+        result.findings = self._summarise(result)
+        return result
+
+    def _summarise(self, result: AttentionResult) -> dict[str, Any]:
+        """Build the light, agent-readable summary stored in ``findings``."""
+        top = result.top_attended_tokens(
+            query_pos=-1, k=self.top_k, layer=self.layer, head=self.head
+        )
+        entropy = result.attention_entropy(layer=self.layer, head=self.head)
+        return {
+            "num_layers": result.num_layers,
+            "num_heads": result.num_heads,
+            "seq_len": result.seq_len,
+            "summary_layer": self.layer,
+            "summary_head": self.head,
+            "top_attended_tokens": [{"token": t, "weight": round(w, 4)} for t, w in top],
+            "mean_attention_entropy": round(float(entropy.mean()), 4),
+        }

evalrx/analyzers/attribution/__init__.py

@@ -0,0 +1,6 @@
+"""Gradient/attention attribution analyzers (white-box; require GRADIENTS)."""
+
+from evalrx.analyzers.attribution.generic_attn import GenericAttentionExplainability
+from evalrx.analyzers.attribution.gradcam import GradCAMAnalyzer
+
+__all__ = ["GradCAMAnalyzer", "GenericAttentionExplainability"]

evalrx/analyzers/attribution/generic_attn.py

@@ -0,0 +1,31 @@
+"""Generic attention-model explainability (Chefer et al., 2021) (Stage 2).
+
+Relevance propagation through attention layers using attention weights AND their
+gradients — hence ``requires={ATTENTION, GRADIENTS}`` (white-box, not black-box,
+correcting the original taxonomy). Built for enc-dec / bi-modal transformers; not
+decoder-only-native.
+
+References:
+- Generic Attention-model Explainability for Interpreting Bi-Modal and Encoder-Decoder Transformers
+  Chefer, Gur & Wolf, ICCV 2021 — arXiv:2103.15679
+- Predecessor — Transformer Interpretability Beyond Attention Visualization,
+  Chefer, Gur & Wolf, CVPR 2021 — arXiv:2012.09838
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+
+
+@register_analyzer("generic_attention")
+class GenericAttentionExplainability(Analyzer):
+    name = "generic_attention"
+    requires = frozenset({Capability.ATTENTION, Capability.GRADIENTS})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def _run(self, model, cases):
+        raise NotImplementedError(
+            "Stage 2: Chefer-style relevance via attention × gradient-of-attention propagation."
+        )

evalrx/analyzers/attribution/gradcam.py

@@ -0,0 +1,30 @@
+"""Grad-CAM on the vision tower (Stage 2).
+
+Gradient-weighted activation maps over the ViT feature map → spatial saliency.
+``requires=GRADIENTS`` (backward pass); VLM vision tower only. Needs a
+``reshape_transform`` for ViT and the per-arch vision-tower target layer.
+
+References:
+- Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization
+  Selvaraju et al., ICCV 2017 — arXiv:1610.02391
+- Code: https://github.com/jacobgil/pytorch-grad-cam
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+
+
+@register_analyzer("gradcam")
+class GradCAMAnalyzer(Analyzer):
+    name = "gradcam"
+    requires = frozenset({Capability.GRADIENTS})
+    applies_to_modalities = frozenset({"image"})
+
+    def _run(self, model, cases):
+        raise NotImplementedError(
+            "Stage 2: register a hook on the vision-tower target block, backward through a "
+            "target logit, weight activations by gradients (pytorch-grad-cam + reshape_transform)."
+        )

evalrx/analyzers/base.py

@@ -0,0 +1,12 @@
+"""Analyzer base re-export — every analyzer subclasses the core estimator contract.
+
+Configure hyper-parameters in ``__init__``, declare ``requires`` (capabilities) and
+``applies_to_modalities``, implement ``_run(model, cases) -> Result``.
+"""
+
+from __future__ import annotations
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.result import Result
+
+__all__ = ["Analyzer", "Result"]

evalrx/analyzers/geometry/__init__.py

@@ -0,0 +1,6 @@
+"""Embedding-geometry analyzers (HIDDEN_STATES). CLIP/SigLIP-tower-scoped — see caveats."""
+
+from evalrx.analyzers.geometry.cka import CKAAnalyzer, linear_cka
+from evalrx.analyzers.geometry.linear_probe import LinearProbeAnalyzer
+
+__all__ = ["CKAAnalyzer", "linear_cka", "LinearProbeAnalyzer"]

evalrx/analyzers/geometry/cka.py

@@ -0,0 +1,70 @@
+"""Linear CKA — representational similarity across layers.
+
+Computes pairwise linear Centered Kernel Alignment between layers' hidden states
+(sequence positions as samples).  ``requires=HIDDEN_STATES``.
+
+Construct-validity caveat: CKA/cosine-geometry methods were validated on CLIP-style
+two-tower encoders. A decoder-only MLLM has no joint two-tower space, so cross-modal
+geometry numbers don't carry the meaning the papers claim — scope this to a frozen
+CLIP/SigLIP tower's activations, or read it only as within-stream layer similarity.
+
+References:
+- Similarity of Neural Network Representations Revisited (linear CKA)
+  Kornblith, Norouzi, Lee & Hinton, ICML 2019 — arXiv:1905.00414
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+def linear_cka(X, Y) -> float:
+    """Linear CKA between two activation matrices ``(n, d1)`` and ``(n, d2)``."""
+
+    X = X.float() - X.float().mean(0, keepdim=True)
+    Y = Y.float() - Y.float().mean(0, keepdim=True)
+    hsic = (Y.T @ X).pow(2).sum()
+    denom = (X.T @ X).norm() * (Y.T @ Y).norm()
+    return float(hsic / denom) if denom > 0 else 0.0
+
+
+@register_analyzer("cka")
+class CKAAnalyzer(Analyzer):
+    """Pairwise linear-CKA matrix across the model's layers for one input."""
+
+    name = "cka"
+    requires = frozenset({Capability.HIDDEN_STATES})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        import torch
+
+        case = cases[0]
+        trace = model.forward(case.inputs, capture={Capability.HIDDEN_STATES})
+        hidden = trace.require(Capability.HIDDEN_STATES)  # list of (seq, dim)
+        L = len(hidden)
+        mat = torch.zeros(L, L)
+        for i in range(L):
+            for j in range(i, L):
+                c = linear_cka(hidden[i], hidden[j])
+                mat[i, j] = mat[j, i] = c
+        off = mat[~torch.eye(L, dtype=torch.bool)]
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"cka_matrix": mat},
+            findings={
+                "n_layers": L,
+                "mean_offdiagonal_cka": round(float(off.mean()), 4),
+                "adjacent_layer_cka": [round(float(mat[i, i + 1]), 4) for i in range(L - 1)],
+                "_caveat": "CKA validated on CLIP two-tower encoders; scope to a tower, not decoder MLLM cross-modal geometry.",
+            },
+        )

evalrx/analyzers/geometry/linear_probe.py

@@ -0,0 +1,157 @@
+"""Linear probing — per-layer representation richness for a target property.
+
+Train a linear classifier on each layer's hidden states (samples + labels) and
+report per-layer HELD-OUT accuracy — where in the stack is the property linearly
+decodable.  Labels come from ``case.label`` (PASS/FAIL), so the probe answers:
+"at which depth do the representations already separate failing from passing
+inputs".  Correlational, not causal.
+
+Implementation notes (no sklearn dependency — torch only):
+- one forward per case (``HIDDEN_STATES``), feature = hidden state at ``pos``;
+- per-layer logistic regression, class-weighted BCE + L2, Adam;
+- stratified 2-fold cross-validation: every reported number and every
+  ``per_case`` probability is out-of-fold (never train-set);
+- balanced accuracy (mean of per-class recalls) — robust to FAIL/PASS skew;
+- too few labelled cases per class -> returns a SKIPPED result with the reason
+  instead of raising (an analyzer crash silently removes the evidence slot
+  from the loop's view; a degraded honest result does not).
+
+References:
+- Understanding intermediate layers using linear classifier probes
+  Alain & Bengio, 2016 — arXiv:1610.01644
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("linear_probe")
+class LinearProbeAnalyzer(Analyzer):
+    """Per-layer linear-probe (held-out, balanced) accuracy for PASS/FAIL."""
+
+    name = "linear_probe"
+    requires = frozenset({Capability.HIDDEN_STATES})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    # max_cases sized for enriched batches (see LogitLensAnalyzer): the held-out
+    # probe needs enough of the scarce FAIL class for a trustworthy accuracy
+    # curve and a powered downstream contrast; one forward per case.
+    def __init__(self, pos: int = -1, max_cases: int = 0, min_per_class: int = 4,
+                 l2: float = 1e-2, epochs: int = 200, seed: int = 0) -> None:
+        super().__init__(pos=pos, max_cases=max_cases, min_per_class=min_per_class,
+                         l2=l2, epochs=epochs, seed=seed)
+
+    # ------------------------------------------------------------------
+    def _fit_predict(self, X_tr, y_tr, X_te):
+        """Class-weighted logistic regression; returns P(fail) on X_te."""
+        import torch
+
+        torch.manual_seed(self.seed)
+        d = X_tr.shape[1]
+        w = torch.zeros(d, requires_grad=True)
+        b = torch.zeros(1, requires_grad=True)
+        n_pos = float(y_tr.sum())
+        pos_weight = torch.tensor((len(y_tr) - n_pos) / max(n_pos, 1.0))
+        opt = torch.optim.Adam([w, b], lr=0.05)
+        loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)
+        for _ in range(self.epochs):
+            opt.zero_grad()
+            loss = loss_fn(X_tr @ w + b, y_tr) + self.l2 * w.pow(2).sum()
+            loss.backward()
+            opt.step()
+        with torch.no_grad():
+            return torch.sigmoid(X_te @ w + b)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        import torch
+
+        from evalrx.core.case import CaseBatch, Label
+
+        # Label-stratified subsample so the capped probe pool keeps enough FAIL
+        # cases (a plain head is mostly PASS on an enriched batch).
+        labelled_batch = CaseBatch([c for c in cases
+                                    if c.label in (Label.PASS, Label.FAIL)])
+        labelled = labelled_batch.stratified_head(self.max_cases)
+        n_fail = sum(1 for c in labelled if c.label == Label.FAIL)
+        n_pass = len(labelled) - n_fail
+
+        def skipped(reason: str) -> Result:
+            return Result(
+                analyzer=self.name, model=repr(model), cases=cases,
+                findings={"skipped": reason, "n_fail": n_fail, "n_pass": n_pass,
+                          "n_layers": None, "per_layer_accuracy": [],
+                          "best_layer": None, "best_accuracy": None, "per_case": []},
+            )
+
+        if min(n_fail, n_pass) < self.min_per_class:
+            return skipped(f"needs >= {self.min_per_class} cases per class "
+                           f"(got fail={n_fail}, pass={n_pass})")
+
+        # -- collect features: one forward per case, all layers at `pos` ----
+        feats, ys, ids = [], [], []
+        for case in labelled:
+            trace = model.forward(case.inputs, capture={Capability.HIDDEN_STATES})
+            hidden = trace.require(Capability.HIDDEN_STATES)
+            feats.append(torch.stack([h[self.pos].detach().float().cpu() for h in hidden]))
+            ys.append(1.0 if case.label == Label.FAIL else 0.0)
+            ids.append(case.id)
+        X = torch.stack(feats)            # (n, L, d)
+        y = torch.tensor(ys)
+        n, n_layers, _ = X.shape
+
+        # -- stratified 2-fold assignment (seeded, deterministic) -----------
+        g = torch.Generator().manual_seed(self.seed)
+        fold = torch.zeros(n, dtype=torch.long)
+        for cls in (0.0, 1.0):
+            idx = [i for i in range(n) if ys[i] == cls]
+            perm = torch.randperm(len(idx), generator=g)
+            for j, p in enumerate(perm):
+                fold[idx[p]] = j % 2
+
+        per_layer_acc, oof = [], torch.zeros(n_layers, n)
+        for layer in range(n_layers):
+            Xl = X[:, layer, :]
+            accs = []
+            for f in (0, 1):
+                tr, te = (fold != f), (fold == f)
+                mu, sd = Xl[tr].mean(0), Xl[tr].std(0).clamp_min(1e-6)
+                p = self._fit_predict((Xl[tr] - mu) / sd, y[tr], (Xl[te] - mu) / sd)
+                oof[layer, te] = p
+                pred = (p > 0.5).float()
+                yt = y[te]
+                rec_f = ((pred == 1) & (yt == 1)).sum() / (yt == 1).sum().clamp_min(1)
+                rec_p = ((pred == 0) & (yt == 0)).sum() / (yt == 0).sum().clamp_min(1)
+                accs.append(float((rec_f + rec_p) / 2))
+            per_layer_acc.append(round(sum(accs) / len(accs), 4))
+
+        best_layer = max(range(n_layers), key=lambda i: per_layer_acc[i])
+        per_case = [{
+            "sample_id": ids[i],
+            "fail_prob_best_layer": round(float(oof[best_layer, i]), 4),
+            "is_fail": bool(ys[i]),
+        } for i in range(n)]
+
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"per_layer_accuracy": per_layer_acc, "per_case": per_case},
+            findings={
+                "n_layers": n_layers, "pos": self.pos,
+                "n_cases": n, "n_fail": n_fail, "n_pass": n_pass,
+                "per_layer_accuracy": per_layer_acc,
+                "best_layer": best_layer,
+                "best_layer_frac": round(best_layer / max(1, n_layers - 1), 4),
+                "best_accuracy": per_layer_acc[best_layer],
+                "chance": 0.5,  # balanced accuracy baseline
+                "per_case": per_case,
+            },
+        )

evalrx/analyzers/hallucination/__init__.py

@@ -0,0 +1,9 @@
+"""Hallucination diagnostics — the dominant MLLM failure mode (BB probes + WB attention)."""
+
+from evalrx.analyzers.hallucination.chair import CHAIRAnalyzer, chair_score
+from evalrx.analyzers.hallucination.opera import OPERAAnalyzer
+from evalrx.analyzers.hallucination.pope import POPEAnalyzer
+from evalrx.analyzers.hallucination.selfcheck import SelfCheckConsistencyAnalyzer
+from evalrx.analyzers.hallucination.vcd import VCDAnalyzer
+
+__all__ = ["POPEAnalyzer", "CHAIRAnalyzer", "chair_score", "OPERAAnalyzer", "VCDAnalyzer", "SelfCheckConsistencyAnalyzer"]

evalrx/analyzers/hallucination/chair.py

@@ -0,0 +1,78 @@
+"""CHAIR — Caption Hallucination Assessment with Image Relevance.
+
+Black-box (``GENERATE``): generate a caption, extract mentioned objects from a
+fixed object vocabulary, and compare to the image's gold objects
+(``metadata["gt_objects"]``).  Reports CHAIR_i (mean fraction of mentioned objects
+that are hallucinated) and CHAIR_s (fraction of captions with ≥1 hallucination).
+``chair_score`` is the reusable per-instance metric.
+
+Paper: "Object Hallucination in Image Captioning"
+       Rohrbach et al., EMNLP 2018 — https://arxiv.org/abs/1809.02156
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Iterable
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+def chair_score(generated_objects: Iterable[str], gt_objects: Iterable[str]) -> dict:
+    """Per-instance CHAIR_i: fraction of mentioned objects that are hallucinated."""
+    gen = {o.lower() for o in generated_objects}
+    gt = {o.lower() for o in gt_objects}
+    hallucinated = sorted(gen - gt)
+    chair_i = len(hallucinated) / len(gen) if gen else 0.0
+    return {"chair_i": round(chair_i, 4), "hallucinated": hallucinated, "n_mentioned": len(gen)}
+
+
+def extract_objects(caption: str, vocab: Iterable[str]) -> list[str]:
+    """Naive object extractor: vocabulary words that appear in the caption."""
+    text = f" {caption.lower()} "
+    return [w for w in vocab if f" {w.lower()} " in text or f" {w.lower()}s " in text]
+
+
+@register_analyzer("chair")
+class CHAIRAnalyzer(Analyzer):
+    """Object-hallucination rate of generated captions vs gold objects."""
+
+    name = "chair"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"image"})
+    # An omni model DECLARES image even on an audio benchmark, so the registry
+    # match alone still offers this on a batch with no images in it.
+    requires_modalities = frozenset({"image"})
+
+    def __init__(self, object_vocab: Iterable[str], gt_key: str = "gt_objects") -> None:
+        super().__init__(object_vocab=list(object_vocab), gt_key=gt_key)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case = []
+        chair_is = []
+        n_with_hallucination = 0
+        for case in cases:
+            caption = model.generate(case.inputs)
+            mentioned = extract_objects(caption, self.object_vocab)
+            gt = case.metadata.get(self.gt_key, [])
+            sc = chair_score(mentioned, gt)
+            per_case.append({"id": case.id, **sc, "mentioned": mentioned})
+            chair_is.append(sc["chair_i"])
+            if sc["hallucinated"]:
+                n_with_hallucination += 1
+        n = len(chair_is)
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            artifacts={"per_case": per_case},
+            findings={
+                "n": n,
+                "chair_i": round(sum(chair_is) / n, 4) if n else None,   # mean per-instance hallucination rate
+                "chair_s": round(n_with_hallucination / n, 4) if n else None,  # fraction of captions w/ a hallucination
+            },
+        )