evalrx 0.1.2__py3-none-any.whl

evalrx/core/spec.py

@@ -0,0 +1,173 @@
+"""ModelSpec — backend-orthogonal, per-model identity knowledge.
+
+The key separation this module enforces:
+
+  * **capabilities come from the BACKEND** (api / vllm_offline / hf_local),
+  * **model identity comes from the SPEC** (this file).
+
+One ``ModelSpec`` for, say, Qwen3-VL drives an API call, a vLLM batch-scoring
+run, *and* an HF-eager attention capture — only the capability set differs.
+``compose(spec, backend)`` (see :mod:`evalrx.models.compose`) combines the
+two into a :class:`~evalrx.core.model.Model`.
+
+This module is intentionally **dependency-free** (no torch / transformers), so it
+imports on the light, pure-API install.  Module *paths* here are HINTS only: the
+white-box backend RESOLVES them at load time against ``named_modules()`` rather
+than trusting them, because they drift across transformers point-releases and
+per-checkpoint wrappers (the doubled-``.model.`` / Llama4-no-``.model`` / v5
+fused-experts traps).  Likewise token ids and merge sizes are read from the live
+``config`` / processor, never hard-coded as values.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+
+class AttnSemantics(str, Enum):
+    """What an attention capture actually MEANS for this architecture.
+
+    Prevents an attention analyzer from blindly assuming a dense ``[B,H,Q,K]``
+    token-attention matrix when the model uses a compressed/fused attention.
+    """
+
+    STANDARD = "standard"      # eager yields a real [B, H, Q, K] token matrix
+    MLA_LATENT = "mla_latent"  # DeepSeek / Kimi: materialised in decompressed latent head space
+    NONE = "none"              # no usable weights (e.g. fused-only vision tower)
+    UNVERIFIED = "unverified"  # e.g. Step3 MFA — probe tensor shape before trusting
+    #: Hybrid stacks (Qwen3.5, Qwen3-Next): most layers are linear/SSM attention
+    #: with NO query-key matrix, and only every k-th layer is full attention.
+    #: Each captured tensor is a real [H, Q, K], but the returned list is SHORTER
+    #: than the model's depth and its positions are not layer numbers — read
+    #: ``config.text_config.layer_types`` to map them back. Anything that
+    #: composes across the stack (attention rollout) is measuring a partial path.
+    HYBRID_SPARSE = "hybrid_sparse"
+
+
+@dataclass(frozen=True)
+class ModulePaths:
+    """Dotted-attribute HINTS for hooking, relative to the loaded top object.
+
+    ``"{i}"`` marks the decoder-layer index.  These are *starting points* for
+    runtime resolution, not ground truth — the hf_local backend discovers the
+    real decoder-layer ``ModuleList`` and derives the rest relative to it.
+    """
+
+    decoder_layers: Optional[str] = None   # hint, e.g. "model.language_model.layers"
+    self_attn: str = "self_attn"           # submodule within a decoder layer
+    mlp: str = "mlp"                        # "mlp" | "feed_forward"; dense vs MoE differ
+    vision_tower: Optional[str] = None      # VLM only; hint, e.g. "model.visual"
+    vision_blocks: Optional[str] = None     # e.g. ".encoder.blocks" — varies a lot per tower
+    router: Optional[str] = None            # MoE routing submodule (relative to layer)
+    experts: Optional[str] = None           # MoE experts; ModuleList OR fused param (v5)
+
+
+@dataclass(frozen=True)
+class VisionSpec:
+    """VLM-only knowledge for locating image tokens and rebuilding the patch grid.
+
+    Every field is a HINT or a *config attribute name* — never a baked value —
+    because image-token ids differ across checkpoints (GLM-4.5V 151363 vs
+    GLM-4.1V 151343) and configs name them differently (``image_token_id`` /
+    ``image_token_index`` / ``media_placeholder_token_id``).
+    """
+
+    # name of the config attribute holding the per-patch placeholder token id:
+    image_token_id_attr: str = "image_token_id"
+    # name of the config path holding the spatial merge size (None -> not applicable):
+    merge_size_attr: Optional[str] = "vision_config.spatial_merge_size"
+    grid_source: str = "grid_thw"           # "grid_thw" | "grid_hw" | "fixed"
+    fixed_tokens_per_tile: Optional[int] = None
+    prefer_mm_token_type_ids: bool = True    # cleanest source of truth when emitted
+
+
+@dataclass(frozen=True)
+class AudioSpec:
+    """Omni-only knowledge for locating audio tokens (Stage-2 white-box capture).
+
+    Symmetric to :class:`VisionSpec`: every field is a HINT or a *config attribute
+    name*, never a baked value, because audio placeholder ids and tower paths drift
+    across checkpoints (Qwen3-Omni's ``thinker.audio_tower`` vs others).  Its mere
+    presence on a spec declares the ``"audio"`` modality; the white-box audio-token
+    map that mirrors :class:`~evalrx.core.tokentype.TokenTypeMap` is Stage-2.
+    """
+
+    # name of the config attribute holding the per-frame audio placeholder token id:
+    audio_token_id_attr: str = "audio_token_id"
+    audio_tower: Optional[str] = None        # hooking HINT, e.g. "thinker.audio_tower"
+    # Qwen3-Omni: whether a video's own audio track is fed alongside its frames
+    # (kept consistent between preprocessing and generation by the processor).
+    use_audio_in_video: bool = True
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    """Per-model identity, reused across every backend.
+
+    Holds NO capabilities — those are the backend's.  Modality is declared by the
+    *components present*, not a class fork: ``vision`` adds ``"image"``, ``audio``
+    adds ``"audio"``, ``video`` adds ``"video"`` (an omni model simply carries more
+    than one).  Pure-LLM specs leave them unset (no TokenTypeMap).  Closed models
+    set ``api_only``.
+    """
+
+    key: str                                  # registry key, e.g. "qwen3-vl-8b-instruct"
+    family: str                               # "qwen3_vl" | "deepseek_v3" | ...
+    model_type: str                           # config.model_type (used to branch)
+    hf_repo: str = ""                         # "" for closed/api-only models
+    auto_class: str = "AutoModelForCausalLM"  # or "AutoModelForImageTextToText"
+    trust_remote_code: bool = False
+    min_transformers: str = "4.51.0"
+    processor_class: str = "AutoProcessor"    # "AutoTokenizer" for text-only
+    chat_template_kwargs: dict = field(default_factory=dict)
+    is_moe: bool = False
+    is_reasoning: bool = False                # emits <think>...</think>
+    tool_calling: bool = False                # chat template renders tools -> grants TOOL_CALLS on local backends
+    eager_required_for_attn: bool = True
+    attn_semantics: AttnSemantics = AttnSemantics.STANDARD
+    module_paths: Optional[ModulePaths] = None  # None -> rely fully on discovery
+    vision: Optional[VisionSpec] = None        # not None -> "image" modality
+    audio: Optional[AudioSpec] = None          # not None -> "audio" modality (omni)
+    video: bool = False                         # True -> "video" modality (omni)
+    api_only: bool = False                      # closed weights -> only the api backend
+    caveats: tuple[str, ...] = ()               # honest, human-readable gotchas
+
+    @property
+    def is_vlm(self) -> bool:
+        """Carries a vision tower — drives the image-token forward path."""
+        return self.vision is not None
+
+    @property
+    def is_omni(self) -> bool:
+        """Handles audio and/or video on top of text (and usually image)."""
+        return self.audio is not None or self.video
+
+    @property
+    def needs_multimodal_encode(self) -> bool:
+        """True when the backend must route through the (image/video/audio) encoder.
+
+        Distinct from ``is_vlm`` alone so an audio-only spec (``vision is
+        None``, e.g. the Qwen3-Omni captioner or a Qwen2-Audio spec) still
+        takes the multimodal encode path in the backend instead of falling
+        through to the plain-text tokenizer and silently dropping its audio.
+        """
+        return self.vision is not None or self.audio is not None
+
+    @property
+    def modalities(self) -> frozenset[str]:
+        """Modalities derived from the components present (analyzers match on this)."""
+        mods = {"text"}
+        if self.vision is not None:
+            mods.add("image")
+        if self.audio is not None:
+            mods.add("audio")
+        if self.video:
+            mods.add("video")
+        return frozenset(mods)
+
+    def __str__(self) -> str:  # pragma: no cover - cosmetic
+        kind = "Omni" if self.is_omni else ("VLM" if self.is_vlm else "LLM")
+        tag = " api-only" if self.api_only else ""
+        return f"ModelSpec({self.key!r}, {kind}, family={self.family}{tag})"

evalrx/core/tokentype.py

@@ -0,0 +1,165 @@
+"""TokenTypeMap — the shared VLM primitive that locates image tokens + the patch grid.
+
+Every white-box VLM analyzer (Grad-CAM, relative-attention, MM-SHAP,
+image-token logit-lens, "what's in the image") needs to know which sequence
+positions are image-patch tokens and how they map to a spatial grid.  Building
+that is genuinely model-specific (dynamic resolution + spatial-merge), so it is
+centralised here and driven by a :class:`~evalrx.core.spec.VisionSpec`.
+
+Design notes that this encodes:
+  * prefer the processor's ``mm_token_type_ids`` when emitted (cleanest), else
+    match ``input_ids == config.<image_token_id_attr>`` — the id is read from the
+    live config (GLM-4.5V 151363 vs GLM-4.1V 151343), never baked in;
+  * the grid is read from processor metadata per ``grid_source``:
+      grid_thw  -> image_grid_thw (PRE-merge patch units) -> post = (h//m, w//m)   [Qwen/GLM]
+      grid_hw   -> image_grid_hws (pre-merge) -> //m                                [Kimi]
+      fixed     -> a fixed token budget per tile                                    [Gemma/Step]
+  * merge size is read from config (``vision_config.spatial_merge_size``), not literal.
+
+This module is torch-tolerant: it accepts torch tensors OR plain lists, importing
+torch nowhere — so it unit-tests on the light install with synthetic inputs.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+def _flat_ids(x: Any) -> list[int]:
+    """Flatten input_ids (torch tensor / np / nested list / 1-D list) to a 1-D int list."""
+    if x is None:
+        return []
+    if hasattr(x, "flatten") and hasattr(x, "tolist"):  # torch / numpy tensor
+        return [int(v) for v in x.flatten().tolist()]
+    if isinstance(x, (list, tuple)):
+        if x and isinstance(x[0], (list, tuple)):  # [[...]] -> first row
+            return [int(v) for v in x[0]]
+        return [int(v) for v in x]
+    return [int(v) for v in list(x)]
+
+
+def _rows2(x: Any) -> list[list[int]]:
+    """Coerce an (n, k) grid spec (tensor / nested list) to a list of int rows."""
+    if x is None:
+        return []
+    if hasattr(x, "tolist"):
+        x = x.tolist()
+    rows = []
+    for row in x:
+        rows.append([int(v) for v in (row if isinstance(row, (list, tuple)) else [row])])
+    return rows
+
+
+def _resolve(obj: Any, dotted: Optional[str]) -> Any:
+    """Resolve a dotted attribute path against a config object; None on miss."""
+    if obj is None or not dotted:
+        return None
+    cur = obj
+    for part in dotted.split("."):
+        cur = getattr(cur, part, None)
+        if cur is None:
+            return None
+    return cur
+
+
+def _enc_get(enc: Any, key: str) -> Any:
+    """Read a key from a processor BatchFeature / dict."""
+    if enc is None:
+        return None
+    if hasattr(enc, "get"):
+        try:
+            return enc.get(key)
+        except Exception:
+            pass
+    return getattr(enc, key, None)
+
+
+@dataclass
+class TokenTypeMap:
+    """Where the image tokens are, and how they map to a spatial grid.
+
+    Attributes:
+        seq_len:    total sequence length.
+        image_pos:  sequence positions that are image-patch tokens (flat, in order).
+        text_pos:   the remaining (text/control) positions.
+        grids:      per image, the POST-merge grid ``(t, h, w)``.
+        patches:    aligned with ``image_pos`` — ``(image_idx, t, row, col)`` for each.
+        image_token_id: the placeholder id used (if matched by id).
+    """
+
+    seq_len: int
+    image_pos: list[int] = field(default_factory=list)
+    text_pos: list[int] = field(default_factory=list)
+    grids: list[tuple[int, int, int]] = field(default_factory=list)
+    patches: list[tuple[int, int, int, int]] = field(default_factory=list)
+    image_token_id: Optional[int] = None
+
+    @property
+    def n_images(self) -> int:
+        return len(self.grids)
+
+    @property
+    def has_image(self) -> bool:
+        return len(self.image_pos) > 0
+
+
+def build_token_type_map(input_ids: Any, enc: Any, config: Any, vision_spec: Any) -> TokenTypeMap:
+    """Construct a :class:`TokenTypeMap` from a forward's encoding + model config + VisionSpec."""
+    ids = _flat_ids(input_ids)
+    seq = len(ids)
+
+    # 1. locate image positions
+    img_id = _resolve(config, vision_spec.image_token_id_attr) if config is not None else None
+    mm = _enc_get(enc, "mm_token_type_ids")
+    if getattr(vision_spec, "prefer_mm_token_type_ids", True) and mm is not None:
+        mm_flat = _flat_ids(mm)
+        image_pos = [i for i, v in enumerate(mm_flat) if v == 1]
+    elif img_id is not None:
+        image_pos = [i for i, t in enumerate(ids) if t == img_id]
+    else:
+        image_pos = []
+    img_set = set(image_pos)
+    text_pos = [i for i in range(seq) if i not in img_set]
+
+    # 2. per-image POST-merge grid
+    merge = _resolve(config, getattr(vision_spec, "merge_size_attr", None)) or 1
+    merge = int(merge) if merge else 1
+    grids: list[tuple[int, int, int]] = []
+    source = getattr(vision_spec, "grid_source", "grid_thw")
+    if source == "grid_thw":
+        for row in _rows2(_enc_get(enc, "image_grid_thw")):
+            t, h, w = (row + [1, 1, 1])[:3]
+            grids.append((int(t), max(int(h) // merge, 1), max(int(w) // merge, 1)))
+    elif source == "grid_hw":
+        for row in _rows2(_enc_get(enc, "image_grid_hws")):
+            h, w = (row + [1, 1])[:2]
+            grids.append((1, max(int(h) // merge, 1), max(int(w) // merge, 1)))
+    elif source == "fixed":
+        per = getattr(vision_spec, "fixed_tokens_per_tile", None)
+        if per:
+            n_tiles = max(len(image_pos) // per, 1) if image_pos else 0
+            side = int(math.isqrt(per))
+            for _ in range(n_tiles):
+                grids.append((1, side, side) if side * side == per else (1, 1, per))
+
+    # 3. map each image position to (image_idx, t, row, col), row-major
+    patches: list[tuple[int, int, int, int]] = []
+    k = 0
+    for img_idx, (t, h, w) in enumerate(grids):
+        for ti in range(t):
+            for r in range(h):
+                for c in range(w):
+                    if k < len(image_pos):
+                        patches.append((img_idx, ti, r, c))
+                        k += 1
+
+    return TokenTypeMap(
+        seq_len=seq,
+        image_pos=image_pos,
+        text_pos=text_pos,
+        grids=grids,
+        patches=patches,
+        image_token_id=int(img_id) if isinstance(img_id, int) else None,
+    )

evalrx/core/tool.py

@@ -0,0 +1,92 @@
+"""Tool / ToolCall / ChatTurn — the value types the agent loop speaks.
+
+Backend-agnostic and torch-free.  A :class:`Tool` is defined ONCE (name +
+JSON-schema params + a python callable) and works on every backend: its
+``to_openai_schema()`` is accepted both by OpenAI-compatible APIs and by
+transformers' ``apply_chat_template(tools=...)``.  A :class:`ChatTurn` is the
+uniform transport from a model handle back to the codec/agent: ``text`` plus, for
+native-tool-calling backends, the backend's raw structured ``tool_calls``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+
+@dataclass
+class Tool:
+    """A callable tool the agent may invoke.
+
+    ``parameters`` is a JSON-schema object describing the args (the standard
+    OpenAI ``function.parameters`` shape).
+    """
+
+    name: str
+    description: str = ""
+    parameters: dict = field(default_factory=lambda: {"type": "object", "properties": {}})
+    fn: Optional[Callable[..., Any]] = None  # the actual implementation (executor calls it)
+
+    def to_openai_schema(self) -> dict:
+        """OpenAI tool schema — also accepted by HF ``apply_chat_template(tools=)``."""
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": self.description,
+                "parameters": self.parameters,
+            },
+        }
+
+
+@dataclass
+class ToolCall:
+    """A normalised tool invocation decoded from a model's output."""
+
+    name: str
+    args: dict = field(default_factory=dict)
+    id: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return {"name": self.name, "args": self.args, "id": self.id}
+
+
+@dataclass
+class ToolResult:
+    """A structured tool return value: model-visible text + optional artifacts.
+
+    ``text`` is what goes back to the model as the tool observation; ``images``
+    (PIL images or paths) are re-injected into the conversation as a follow-up
+    multimodal message so vision models can look at what the tool produced
+    (zoom crops, annotated views).  ``meta`` is host-side detail (saved paths,
+    coordinate mode, timings) recorded on the trajectory step, never shown to
+    the model.  Tools may still return a plain value; ``str()`` is the
+    observation either way.
+    """
+
+    text: str = ""
+    images: list = field(default_factory=list)
+    meta: dict = field(default_factory=dict)
+
+    def __str__(self) -> str:
+        return self.text
+
+
+@dataclass
+class ChatTurn:
+    """One model turn in an agent loop: assistant text + optional native tool calls.
+
+    ``raw_tool_calls`` holds the BACKEND-NATIVE structured calls (e.g. OpenAI's
+    ``[{"id","function":{"name","arguments"}}]``) when the backend does native
+    tool-calling; it is ``None`` for template-based backends, where the call is
+    embedded in ``text`` and the codec parses it out.
+
+    ``usage`` carries per-turn accounting when the backend reports it
+    (``{"prompt_tokens": ..., "completion_tokens": ...}``); the agent loop
+    copies it into the step's ``span`` so trajectories carry cost evidence.
+    """
+
+    text: str = ""
+    raw_tool_calls: Optional[list] = None
+    finish_reason: Optional[str] = None
+    usage: Optional[dict] = None

evalrx/datasets/__init__.py

@@ -0,0 +1,41 @@
+"""Dataset loaders → CaseBatch, plus simple answer verifiers.
+
+Text QA: ``LLMQADataset``.  Image+text QA: ``VLMQADataset`` and the
+``TextVQASizeDataset`` / ``Spatial457Dataset`` / ``VQARADDataset`` benchmarks.
+``PureQADataset`` is a back-compat alias of ``LLMQADataset``.
+"""
+
+from evalrx.datasets.base import (
+    Dataset,
+    cases_from_records,
+    contains_answer,
+    exact_match,
+    normalize,
+    read_jsonl,
+)
+from evalrx.datasets.gui_os import GUIOSDataset
+from evalrx.datasets.llm_qa import LLMQADataset, PureQADataset
+from evalrx.datasets.vlm_qa import (
+    Spatial457Dataset,
+    TextVQASizeDataset,
+    VLMQADataset,
+    VQARADDataset,
+)
+from evalrx.datasets.web_search_qa import WebSearchQADataset
+
+__all__ = [
+    "Dataset",
+    "LLMQADataset",
+    "VLMQADataset",
+    "TextVQASizeDataset",
+    "Spatial457Dataset",
+    "VQARADDataset",
+    "PureQADataset",
+    "WebSearchQADataset",
+    "GUIOSDataset",
+    "cases_from_records",
+    "read_jsonl",
+    "exact_match",
+    "contains_answer",
+    "normalize",
+]

evalrx/datasets/base.py

@@ -0,0 +1,68 @@
+"""Dataset base — loaders produce a CaseBatch; plus simple answer verifiers.
+
+A dataset is just a ``load() -> CaseBatch`` of :class:`FailureCase` (the unit
+analyzers consume).  ``cases_from_records`` maps plain dict records (question /
+answer / image / extra metadata) into cases; the verifiers give A/B strategies a
+``case -> bool success`` signal without pulling in a heavy metric library.
+"""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Iterable
+
+from evalrx.core.case import CaseBatch, FailureCase, Inputs, Provenance, Source
+
+
+# ---------------- verifiers ----------------
+def normalize(s) -> str:
+    return " ".join(str(s).strip().lower().split())
+
+
+def exact_match(prediction, gold) -> bool:
+    return normalize(prediction) == normalize(gold)
+
+
+def contains_answer(prediction, gold) -> bool:
+    """True if the (normalised) gold answer appears in the prediction."""
+    return bool(str(gold).strip()) and normalize(gold) in normalize(prediction)
+
+
+# ---------------- record -> cases ----------------
+def cases_from_records(
+    records: Iterable[dict],
+    *,
+    prompt_key: str = "question",
+    answer_key: str = "answer",
+    image_key: str = "image",
+    tags: set[str] | None = None,
+    source: Source = Source.DATASET,
+) -> CaseBatch:
+    """Build a :class:`CaseBatch` from dict records; the full record is kept in metadata."""
+    out = CaseBatch()
+    for rec in records:
+        out.append(
+            FailureCase(
+                inputs=Inputs(prompt=str(rec.get(prompt_key, "")), image=rec.get(image_key)),
+                expected=rec.get(answer_key),
+                tags=set(tags or ()),
+                provenance=Provenance(source=source),
+                metadata=dict(rec),
+            )
+        )
+    return out
+
+
+def read_jsonl(path: str | Path) -> list[dict]:
+    with open(path) as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+class Dataset(ABC):
+    """A benchmark loader: ``load() -> CaseBatch``."""
+
+    @abstractmethod
+    def load(self) -> CaseBatch:  # pragma: no cover - interface
+        ...

evalrx/datasets/gui_os.py

@@ -0,0 +1,52 @@
+"""GUI / OS task loader → CaseBatch (agent tasks scored on final env state).
+
+Records carry a ``goal`` (the instruction), an optional ``screenshot`` image, and
+an ``expected_state`` (the annotated goal state — τ-bench style, compared to the
+final env state rather than string-matching an answer).  Tagged ``gui_os``;
+these cases are typically driven by an Agent and analysed at the trajectory level.
+"""
+
+from __future__ import annotations
+
+from evalrx.core.case import CaseBatch
+from evalrx.datasets.base import Dataset, cases_from_records, read_jsonl
+
+_SAMPLE = [
+    {"goal": "Open Settings and enable dark mode.", "expected_state": {"dark_mode": True}, "app": "settings"},
+    {"goal": "Create a new file named notes.txt on the desktop.",
+     "expected_state": {"file_exists": "~/Desktop/notes.txt"}, "app": "files"},
+]
+
+
+class GUIOSDataset(Dataset):
+    """GUI / OS-level agent tasks (goal + optional screenshot + expected state)."""
+
+    def __init__(
+        self,
+        records: list[dict] | None = None,
+        path: str | None = None,
+        *,
+        prompt_key: str = "goal",
+        image_key: str = "screenshot",
+    ) -> None:
+        self._records = records
+        self._path = path
+        self._keys = dict(prompt_key=prompt_key, answer_key="expected_state", image_key=image_key)
+
+    @classmethod
+    def from_records(cls, records: list[dict], **keys) -> "GUIOSDataset":
+        return cls(records=records, **keys)
+
+    @classmethod
+    def from_jsonl(cls, path: str, **keys) -> "GUIOSDataset":
+        return cls(path=path, **keys)
+
+    @classmethod
+    def sample(cls) -> "GUIOSDataset":
+        return cls(records=_SAMPLE)
+
+    def load(self) -> CaseBatch:
+        records = self._records if self._records is not None else (
+            read_jsonl(self._path) if self._path else _SAMPLE
+        )
+        return cases_from_records(records, tags={"gui_os"}, **self._keys)

evalrx/datasets/llm_qa.py

@@ -0,0 +1,57 @@
+"""LLM (text) QA loader → CaseBatch.
+
+Text-only question answering.  Source-agnostic: in-memory records, a JSONL path,
+or the built-in sample.  ``expected`` carries the gold answer; pair it with
+:func:`evalrx.datasets.base.contains_answer` to turn a model output into a
+success bool for A/B comparisons.  (For image+text QA use :mod:`evalrx.datasets.vlm_qa`.)
+"""
+
+from __future__ import annotations
+
+from evalrx.core.case import CaseBatch
+from evalrx.datasets.base import Dataset, cases_from_records, read_jsonl
+
+_SAMPLE = [
+    {"question": "What is the capital of France?", "answer": "Paris", "difficulty": "easy"},
+    {"question": "What is 17 multiplied by 3?", "answer": "51", "difficulty": "easy"},
+    {"question": "Who wrote Romeo and Juliet?", "answer": "Shakespeare", "difficulty": "easy"},
+    {"question": "What is the chemical symbol for gold?", "answer": "Au", "difficulty": "medium"},
+]
+
+
+class LLMQADataset(Dataset):
+    """Text question-answering loader."""
+
+    def __init__(
+        self,
+        records: list[dict] | None = None,
+        path: str | None = None,
+        *,
+        prompt_key: str = "question",
+        answer_key: str = "answer",
+    ) -> None:
+        self._records = records
+        self._path = path
+        self._keys = dict(prompt_key=prompt_key, answer_key=answer_key)
+
+    @classmethod
+    def from_records(cls, records: list[dict], **keys) -> "LLMQADataset":
+        return cls(records=records, **keys)
+
+    @classmethod
+    def from_jsonl(cls, path: str, **keys) -> "LLMQADataset":
+        return cls(path=path, **keys)
+
+    @classmethod
+    def sample(cls) -> "LLMQADataset":
+        return cls(records=_SAMPLE)
+
+    def load(self) -> CaseBatch:
+        records = self._records if self._records is not None else (
+            read_jsonl(self._path) if self._path else _SAMPLE
+        )
+        return cases_from_records(records, tags={"llm_qa"}, **self._keys)
+
+
+# Back-compat: the old combined name. Prefer LLMQADataset (text) / VLMQADataset (image+text).
+PureQADataset = LLMQADataset

evalrx/datasets/pure_qa.py

@@ -0,0 +1,12 @@
+"""Back-compat shim — ``pure_qa`` was split into ``llm_qa`` (text) and ``vlm_qa`` (image+text).
+
+``PureQADataset`` remains importable here as an alias of
+:class:`~evalrx.datasets.llm_qa.LLMQADataset`.  Prefer the explicit names:
+``LLMQADataset`` for text QA, ``VLMQADataset`` / ``Spatial457Dataset`` for image+text.
+"""
+
+from __future__ import annotations
+
+from evalrx.datasets.llm_qa import LLMQADataset, PureQADataset
+
+__all__ = ["PureQADataset", "LLMQADataset"]