anyjev 0.0.1__py3-none-any.whl

anyjev/__init__.py

@@ -0,0 +1,14 @@
+"""AnyJev: turn any causal LLM into a Jev-style decision model.
+
+Typed decisions (choice / score / noul) with probabilities, read from the
+model's next-token logits in one prefill, no generation. Training-free
+debiasing (L0) is on by default; post-hoc calibration (L1) when you have labels.
+
+Not affiliated with, endorsed by, or derived from TypeSafe AI or Jev.
+"""
+from anyjev.decider import Decider
+from anyjev.question import Question
+from anyjev.result import Decision, DecisionSet
+
+__all__ = ["Question", "Decision", "DecisionSet", "Decider"]
+__version__ = "0.0.1"

anyjev/backends/__init__.py

@@ -0,0 +1,3 @@
+from anyjev.backends.base import Backend
+
+__all__ = ["Backend"]

anyjev/backends/base.py

@@ -0,0 +1,21 @@
+"""Backend protocol. A backend does exactly one thing: given rendered prompts,
+return the next-token log-probabilities of the requested token ids.
+Everything else (debiasing, calibration, abstention) lives above it."""
+from __future__ import annotations
+
+from typing import Any, List, Protocol, Sequence
+
+import numpy as np
+
+
+class Backend(Protocol):
+    tokenizer: Any   # must provide .encode(text, add_special_tokens=False) and optionally .chat_template
+    name: str
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        """For prompt i, return log p(token | prompt_i) for each id in token_ids[i],
+        taken from the full-vocabulary log-softmax at the last position.
+        Decision mode never samples; a backend that cannot expose restricted
+        next-token logits is not a supported backend."""
+        ...

anyjev/backends/fake.py

@@ -0,0 +1,94 @@
+"""A synthetic backend with known, injectable biases. Used by the unit tests
+and by the docs to show what L0 removes. It parses the prompts that
+anyjev.readout builds, so it exercises the real prompt path."""
+from __future__ import annotations
+
+import re
+from typing import Callable, Dict, List, Optional, Sequence
+
+import numpy as np
+
+from anyjev.calibrate.contextual import DEFAULT_PROBES
+
+_OPT_LINE = re.compile(r"^([A-Z]|\d+)\. (.*)$")
+_NOUL_LINE = re.compile(r"^Answer (Yes|No) or (Yes|No)\.$")
+
+
+class FakeTokenizer:
+    chat_template = None
+
+    def __init__(self):
+        self._vocab: Dict[str, int] = {}
+
+    def encode(self, text: str, add_special_tokens: bool = False) -> List[int]:
+        # every label we care about is one token; anything else is "long"
+        if re.fullmatch(r" ?([A-Z]|\d|Yes|No)", text):
+            key = text.strip()
+            if key not in self._vocab:
+                self._vocab[key] = 1000 + len(self._vocab)
+            return [self._vocab[key]]
+        return [1, 2]
+
+    def id_to_label(self, tid: int) -> str:
+        for k, v in self._vocab.items():
+            if v == tid:
+                return k
+        raise KeyError(tid)
+
+
+class FakeBackend:
+    """logit(position j, option o) = content(state, o) + position_bias[j] + label_prior[label_j]
+
+    content is 0 on content-free probes, so the prior is exactly the bias term.
+    """
+
+    def __init__(self, content: Callable[[str, str], float],
+                 position_bias: Optional[Sequence[float]] = None,
+                 label_prior: Optional[Dict[str, float]] = None,
+                 temperature: float = 1.0):
+        self.name = "fake"
+        self.tokenizer = FakeTokenizer()
+        self.content = content
+        self.position_bias = list(position_bias or [])
+        self.label_prior = dict(label_prior or {})
+        self.temperature = temperature
+        self.calls = 0
+        self.prompts_seen = 0
+
+    def _parse(self, prompt: str):
+        state = prompt.split("State:\n", 1)[1].split("\n\nQuestion:", 1)[0]
+        if state == "(empty)":
+            state = ""
+        labels, options = [], []
+        for line in prompt.splitlines():
+            m = _OPT_LINE.match(line)
+            if m:
+                labels.append(m.group(1))
+                options.append(m.group(2))
+            m = _NOUL_LINE.match(line)
+            if m:
+                labels = [m.group(1), m.group(2)]
+                options = list(labels)
+        return state, labels, options
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        self.calls += 1
+        self.prompts_seen += len(prompts)
+        out = []
+        for prompt, ids in zip(prompts, token_ids):
+            state, labels, options = self._parse(prompt)
+            is_probe = state in DEFAULT_PROBES
+            logits = np.zeros(len(ids))
+            by_label = {self.tokenizer.id_to_label(t): k for k, t in enumerate(ids)}
+            for j, (lab, opt) in enumerate(zip(labels, options)):
+                z = 0.0 if is_probe else self.content(state, opt)
+                if j < len(self.position_bias):
+                    z += self.position_bias[j]
+                z += self.label_prior.get(lab, 0.0)
+                logits[by_label[lab]] = z / self.temperature
+            # full-vocab log-softmax: pretend a bit of mass lives elsewhere
+            z = np.concatenate([logits, [-5.0]])
+            lp = z - np.log(np.exp(z - z.max()).sum()) - z.max()
+            out.append(lp[:-1])
+        return out

anyjev/backends/hf.py

@@ -0,0 +1,52 @@
+"""transformers backend: single forward per prompt batch, logits at the last position."""
+from __future__ import annotations
+
+from typing import List, Optional, Sequence
+
+import numpy as np
+
+
+class HFBackend:
+    def __init__(self, model_name: str, device: str = "cuda", dtype: str = "bfloat16",
+                 batch_size: int = 16, trust_remote_code: bool = False, revision: Optional[str] = None):
+        import torch
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+
+        self.name = model_name
+        self.batch_size = batch_size
+        self.device = device
+        self.revision = revision
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code,
+                                                       revision=revision)
+        self.tokenizer.padding_side = "left"
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        torch_dtype = getattr(torch, dtype) if dtype != "auto" else "auto"
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_name, torch_dtype=torch_dtype, device_map=device,
+            trust_remote_code=trust_remote_code, revision=revision)
+        self.model.eval()
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        import torch
+
+        n = len(prompts)
+        # sort by length to reduce padding, restore order at the end
+        lengths = [len(self.tokenizer.encode(p, add_special_tokens=False)) for p in prompts]
+        order = sorted(range(n), key=lambda i: lengths[i])
+        out: List[Optional[np.ndarray]] = [None] * n
+        for start in range(0, n, self.batch_size):
+            idx = order[start:start + self.batch_size]
+            enc = self.tokenizer([prompts[i] for i in idx], return_tensors="pt",
+                                 padding=True, add_special_tokens=False)
+            enc = {k: v.to(self.model.device) for k, v in enc.items()}
+            # explicit position ids so left padding does not shift positions
+            pos = (enc["attention_mask"].cumsum(-1) - 1).clamp(min=0)
+            with torch.no_grad():
+                logits = self.model(**enc, position_ids=pos).logits[:, -1, :].float()
+            lp = torch.log_softmax(logits, dim=-1)
+            for row, i in enumerate(idx):
+                ids = torch.as_tensor(list(token_ids[i]), device=lp.device)
+                out[i] = lp[row, ids].cpu().numpy().astype(np.float64)
+        return out  # type: ignore[return-value]

anyjev/backends/vllm.py

@@ -0,0 +1,58 @@
+"""vLLM backend over the OpenAI-compatible server.
+
+One request per prompt with `max_tokens=1`, `allowed_token_ids` restricted to
+the label tokens and `logprobs=K`. vLLM reports logprobs after its logit
+processors, so the K entries are exactly the labels, normalized over them.
+Prefix caching on the server makes the K permutations of one state cheap.
+
+    vllm serve Qwen/Qwen3-8B --enable-prefix-caching
+    Decider(VLLMBackend("http://localhost:8000", "Qwen/Qwen3-8B"))
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import json
+import urllib.request
+from typing import List, Sequence
+
+import numpy as np
+
+
+class VLLMBackend:
+    def __init__(self, base_url: str, model: str, tokenizer_name: str | None = None,
+                 api_key: str = "EMPTY", workers: int = 16, timeout: float = 120.0):
+        from transformers import AutoTokenizer
+
+        self.base_url = base_url.rstrip("/")
+        self.name = model
+        self.api_key = api_key
+        self.workers = workers
+        self.timeout = timeout
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name or model)
+
+    def _one(self, prompt: str, ids: Sequence[int]) -> np.ndarray:
+        body = {
+            "model": self.name, "prompt": prompt, "max_tokens": 1, "temperature": 0.0,
+            "logprobs": len(ids), "allowed_token_ids": list(ids),
+        }
+        req = urllib.request.Request(
+            self.base_url + "/v1/completions", data=json.dumps(body).encode(),
+            headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"})
+        with urllib.request.urlopen(req, timeout=self.timeout) as r:
+            out = json.load(r)
+        top = out["choices"][0]["logprobs"]["top_logprobs"][0]   # {token_str: logprob}
+        # map back by token id: decode each id the same way the server renders it
+        by_id = {}
+        for tid in ids:
+            tok = self.tokenizer.decode([tid])
+            conv = self.tokenizer.convert_ids_to_tokens(tid)
+            for key in (tok, conv):
+                if key in top:
+                    by_id[tid] = float(top[key])
+                    break
+        return np.array([by_id.get(tid, -30.0) for tid in ids], dtype=np.float64)
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        with cf.ThreadPoolExecutor(self.workers) as ex:
+            return list(ex.map(self._one, prompts, token_ids))

anyjev/calibrate/__init__.py

@@ -0,0 +1,6 @@
+from anyjev.calibrate.contextual import apply_contextual, batch_prior, content_free_prior
+from anyjev.calibrate.permute import cyclic_shifts, flip_rate_across_perms, marginalize
+from anyjev.calibrate.posthoc import TemperatureScaler
+
+__all__ = ["content_free_prior", "batch_prior", "apply_contextual", "cyclic_shifts", "marginalize",
+           "flip_rate_across_perms", "TemperatureScaler"]

anyjev/calibrate/contextual.py

@@ -0,0 +1,47 @@
+"""L0 prior estimation and correction.
+
+Two ways to estimate the model's prior over the labels, both label-free:
+
+  batch (Zhou et al., ICLR 2024, default): the mean predicted distribution
+      over a batch of real inputs, kept running per question across calls.
+      Low variance in the bench: +1 to +2 accuracy points and a large ECE
+      improvement on every model; needs a handful of inputs before it kicks
+      in (min_prior_n), and assumes the label marginal is not extreme.
+  content_free (Zhao et al., ICML 2021, opt-in): the distribution on
+      content-free inputs ("N/A", "", "[MASK]"). Depends only on the question,
+      cached per question. High variance in the bench: +8 to +12 points on a
+      prompt-injection noul, but -3 on ordinal scores and -9 on one model's
+      noul questions, because for some questions the model's answer to an
+      empty input is an honest answer, not a label prior.
+
+Correction is the same in both cases: divide by the prior and renormalize.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+DEFAULT_PROBES = ("N/A", "", "[MASK]")
+EPS = 1e-8
+
+
+def content_free_prior(cf_probs: np.ndarray) -> np.ndarray:
+    """cf_probs: [..., C, K] distributions over K positions for C probes.
+    Returns the mean prior [..., K]."""
+    cf = np.asarray(cf_probs, dtype=np.float64)
+    prior = cf.mean(axis=-2)
+    prior = np.clip(prior, EPS, None)
+    return prior / prior.sum(axis=-1, keepdims=True)
+
+
+def batch_prior(p_batch: np.ndarray) -> np.ndarray:
+    """p_batch: [N, ..., K] distributions over real inputs. Returns [..., K]."""
+    p = np.asarray(p_batch, dtype=np.float64).mean(axis=0)
+    p = np.clip(p, EPS, None)
+    return p / p.sum(axis=-1, keepdims=True)
+
+
+def apply_contextual(p_raw: np.ndarray, prior: np.ndarray) -> np.ndarray:
+    """p_raw, prior: [..., K]. Returns normalize(p_raw / prior)."""
+    p = np.asarray(p_raw, dtype=np.float64) / np.clip(np.asarray(prior, dtype=np.float64), EPS, None)
+    p = np.clip(p, EPS, None)
+    return p / p.sum(axis=-1, keepdims=True)

anyjev/calibrate/permute.py

@@ -0,0 +1,58 @@
+"""L0: permutation marginalization for position bias (Zheng et al., ICLR 2024).
+
+Show the options in K cyclic shifts so every option occupies every position
+once, then combine the per-option probabilities across shifts.
+
+Two ways to combine:
+  "logmean": average log-probabilities (geometric mean), then renormalize.
+             If the bias is additive in logit space, logit(i at pos j) = c_i + b_j,
+             the per-option average is c_i + mean(b) - mean(log Z_s), so the
+             result is softmax(c) exactly: the position bias is removed and the
+             result is invariant to how the options were originally listed.
+  "mean":    average probabilities, the form used in the paper. Only invariant
+             to cyclic rotations of the original list; kept for comparison.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Sequence
+
+import numpy as np
+
+EPS = 1e-12
+
+
+def cyclic_shifts(k: int, max_permutations: Optional[int] = None) -> List[List[int]]:
+    """perm[j] = original option index shown at position j."""
+    n = k if max_permutations is None else max(1, min(k, max_permutations))
+    return [[(j + s) % k for j in range(k)] for s in range(n)]
+
+
+def marginalize(p_by_perm: np.ndarray, perms: Sequence[Sequence[int]],
+                combine: str = "logmean") -> np.ndarray:
+    """p_by_perm: [P, K] distributions indexed by *position*.
+    Returns [K] distribution indexed by original option."""
+    p_by_perm = np.asarray(p_by_perm, dtype=np.float64)
+    P, K = p_by_perm.shape
+    per_option = np.zeros((P, K))
+    for s, perm in enumerate(perms):
+        for j, i in enumerate(perm):
+            per_option[s, i] = p_by_perm[s, j]
+    if combine == "logmean":
+        z = np.log(np.clip(per_option, EPS, None)).mean(axis=0)
+        z = z - z.max()
+        out = np.exp(z)
+    elif combine == "mean":
+        out = per_option.mean(axis=0)
+    else:
+        raise ValueError("combine must be 'logmean' or 'mean'")
+    return out / out.sum()
+
+
+def flip_rate_across_perms(p_by_perm: np.ndarray, perms: Sequence[Sequence[int]]) -> float:
+    """Fraction of permutations whose argmax (in option space) disagrees with
+    the first permutation's argmax. 0.0 means order-invariant on this item."""
+    p_by_perm = np.asarray(p_by_perm)
+    winners = [perm[int(np.argmax(p_by_perm[s]))] for s, perm in enumerate(perms)]
+    if len(winners) <= 1:
+        return 0.0
+    return float(np.mean([w != winners[0] for w in winners[1:]]))

anyjev/calibrate/posthoc.py

@@ -0,0 +1,60 @@
+"""L1: temperature scaling (Guo et al., ICML 2017) fit on a labeled set.
+
+Artifacts are small dicts keyed by (model, question key); reusing one across
+models is a user error and is not detected here.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Sequence
+
+import numpy as np
+
+EPS = 1e-12
+
+
+def _log_softmax(z: np.ndarray) -> np.ndarray:
+    z = z - z.max(axis=-1, keepdims=True)
+    return z - np.log(np.exp(z).sum(axis=-1, keepdims=True))
+
+
+def nll(logits: np.ndarray, labels: Sequence[int]) -> float:
+    lp = _log_softmax(np.asarray(logits, dtype=np.float64))
+    return float(-lp[np.arange(len(labels)), np.asarray(labels)].mean())
+
+
+@dataclass
+class TemperatureScaler:
+    temperature: float = 1.0
+
+    @classmethod
+    def fit(cls, probs: np.ndarray, labels: Sequence[int],
+            log_t_range=(-3.0, 3.0), iters: int = 60) -> "TemperatureScaler":
+        """Golden-section search on log T minimizing NLL. probs: [N, K]."""
+        logits = np.log(np.clip(np.asarray(probs, dtype=np.float64), EPS, None))
+        lo, hi = log_t_range
+        phi = (np.sqrt(5) - 1) / 2
+        a, b = lo, hi
+        c, d = b - phi * (b - a), a + phi * (b - a)
+        fc, fd = nll(logits / np.exp(c), labels), nll(logits / np.exp(d), labels)
+        for _ in range(iters):
+            if fc < fd:
+                b, d, fd = d, c, fc
+                c = b - phi * (b - a)
+                fc = nll(logits / np.exp(c), labels)
+            else:
+                a, c, fc = c, d, fd
+                d = a + phi * (b - a)
+                fd = nll(logits / np.exp(d), labels)
+        return cls(temperature=float(np.exp((a + b) / 2)))
+
+    def apply(self, probs: np.ndarray) -> np.ndarray:
+        logits = np.log(np.clip(np.asarray(probs, dtype=np.float64), EPS, None))
+        return np.exp(_log_softmax(logits / self.temperature))
+
+    def to_dict(self) -> Dict[str, float]:
+        return {"method": "temperature", "temperature": self.temperature}
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> "TemperatureScaler":
+        return cls(temperature=float(d["temperature"]))

anyjev/decider.py

@@ -0,0 +1,236 @@
+"""Decider: the pipeline from (state, questions) to leveled decisions.
+
+raw : one prompt, one permutation, restricted softmax. What every clone does.
+L0  : permutation marginalization + label-free prior correction (batch mean
+      by default, content-free probes optional). Zero labels.
+L1  : L0, then a post-hoc calibrator fit on a labeled set for this question.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from anyjev.calibrate.contextual import (
+    DEFAULT_PROBES,
+    apply_contextual,
+    batch_prior,
+    content_free_prior,
+)
+from anyjev.calibrate.permute import cyclic_shifts, flip_rate_across_perms, marginalize
+from anyjev.calibrate.posthoc import TemperatureScaler
+from anyjev.question import Question
+from anyjev.readout import (
+    DEFAULT_SYSTEM,
+    answer_labels,
+    build_prompt,
+    label_ids_for_perm,
+    map_label_tokens,
+    render_chat,
+)
+from anyjev.result import Decision, DecisionSet
+from anyjev.state import render_state
+
+LEVELS = ("raw", "L0", "L1")
+PRIORS = ("batch", "content_free", "none")
+
+
+def _softmax(lp: np.ndarray) -> np.ndarray:
+    z = np.asarray(lp, dtype=np.float64)
+    z = z - z.max()
+    p = np.exp(z)
+    return p / p.sum()
+
+
+class Decider:
+    def __init__(self, backend, *, level: str = "L0", prior: str = "batch", min_prior_n: int = 8,
+                 max_permutations: Optional[int] = None, combine: str = "logmean",
+                 cf_probes: Sequence[str] = DEFAULT_PROBES, record_content_free: bool = False,
+                 system: str = DEFAULT_SYSTEM):
+        if level not in LEVELS:
+            raise ValueError(f"level must be one of {LEVELS}")
+        if prior not in PRIORS:
+            raise ValueError(f"prior must be one of {PRIORS}")
+        self.backend = backend
+        self.level = level
+        self.prior = prior
+        self.min_prior_n = min_prior_n
+        self.max_permutations = max_permutations
+        self.combine = combine
+        self.cf_probes = tuple(cf_probes)
+        self.record_content_free = record_content_free
+        self.system = system
+        self._label_ids: Dict[tuple, List[int]] = {}
+        self._artifacts: Dict[str, TemperatureScaler] = {}
+        self._running: Dict[str, Tuple[np.ndarray, int]] = {}   # q.key -> (sum p_pos_raw [P,K], n)
+        self._cf_cache: Dict[str, np.ndarray] = {}                # q.key -> cf prior [P,K]
+
+    # ---- public -------------------------------------------------------
+    def decide(self, state: Any, questions: Sequence[Question], level: Optional[str] = None) -> DecisionSet:
+        level = level or self.level
+        decs = self._run([state], list(questions), level)
+        return DecisionSet([decs[(0, qi)] for qi in range(len(questions))], level)
+
+    def decide_batch(self, states: Sequence[Any], question: Question,
+                     level: Optional[str] = None) -> List[Decision]:
+        """Many states, one question. The bench path, and the best path for
+        batch prior estimation."""
+        level = level or self.level
+        decs = self._run(list(states), [question], level)
+        return [decs[(si, 0)] for si in range(len(states))]
+
+    def calibrate(self, question: Question, states: Sequence[Any], labels: Sequence[int],
+                  level: str = "L1") -> Dict[str, Any]:
+        """Fit an L1 artifact for this question on labeled states. labels are
+        option indices. Returns the artifact dict (store it; it is per model)."""
+        if level != "L1":
+            raise ValueError("only L1 calibration is implemented")
+        decs = self.decide_batch(states, question, level="L0")
+        probs = np.stack([d.probs for d in decs])
+        scaler = TemperatureScaler.fit(probs, labels)
+        self._artifacts[question.key] = scaler
+        return {"model": self.backend.name, "question": question.key, **scaler.to_dict()}
+
+    def load_artifact(self, question: Question, artifact: Dict[str, Any]) -> None:
+        if artifact.get("model") not in (None, self.backend.name):
+            raise ValueError(f"artifact was fit on {artifact['model']}, backend is {self.backend.name}")
+        self._artifacts[question.key] = TemperatureScaler.from_dict(artifact)
+
+    def export_artifacts(self) -> Dict[str, Any]:
+        """Every L1 artifact this decider holds, keyed by question hash. JSON-serializable."""
+        return {"model": self.backend.name,
+                "artifacts": {k: {"model": self.backend.name, "question": k, **v.to_dict()}
+                              for k, v in self._artifacts.items()}}
+
+    def save_artifacts(self, path: str) -> None:
+        import json
+        with open(path, "w") as f:
+            json.dump(self.export_artifacts(), f, indent=1)
+
+    def load_artifacts(self, path_or_dict) -> int:
+        """Load artifacts saved by save_artifacts. Refuses artifacts fit on another model."""
+        import json
+        d = path_or_dict if isinstance(path_or_dict, dict) else json.load(open(path_or_dict))
+        if d.get("model") not in (None, self.backend.name):
+            raise ValueError(f"artifacts were fit on {d['model']}, backend is {self.backend.name}")
+        for key, art in d["artifacts"].items():
+            self._artifacts[key] = TemperatureScaler.from_dict(art)
+        return len(d["artifacts"])
+
+    def running_prior(self, question: Question) -> Optional[np.ndarray]:
+        """The batch prior accumulated so far for this question, [P, K] by position, or None."""
+        entry = self._running.get(question.key)
+        if entry is None or entry[1] < self.min_prior_n:
+            return None
+        return batch_prior(entry[0][None] / entry[1])
+
+    # ---- internals ----------------------------------------------------
+    def _ids_for(self, q: Question) -> List[int]:
+        key = (q.kind, q.k)
+        if key not in self._label_ids:
+            self._label_ids[key] = map_label_tokens(self.backend.tokenizer, answer_labels(q))
+        return self._label_ids[key]
+
+    def _perms(self, q: Question, level: str) -> List[List[int]]:
+        if level == "raw" or q.ordered:
+            return [list(range(q.k))]
+        if q.kind == "noul":
+            return [[0, 1], [1, 0]]
+        return cyclic_shifts(q.k, self.max_permutations)
+
+    def _run(self, states: List[Any], questions: List[Question], level: str) -> Dict[tuple, Decision]:
+        if level not in LEVELS:
+            raise ValueError(f"level must be one of {LEVELS}")
+        tok = self.backend.tokenizer
+        state_texts = [render_state(s) for s in states]
+        want_cf = level != "raw" and (self.prior == "content_free" or self.record_content_free)
+
+        # 1. collect every prompt once (content-free probes are shared across states)
+        prompt_index: Dict[str, int] = {}
+        prompt_ids: List[List[int]] = []
+
+        def add(text: str, ids: List[int]) -> int:
+            if text not in prompt_index:
+                prompt_index[text] = len(prompt_ids)
+                prompt_ids.append(ids)
+            return prompt_index[text]
+
+        plan = {}
+        for qi, q in enumerate(questions):
+            ids = self._ids_for(q)
+            perms = self._perms(q, level)
+            cf_rows = []
+            perm_ids = [label_ids_for_perm(q, ids, perm) for perm in perms]
+            if want_cf and q.key not in self._cf_cache:
+                for perm, pids in zip(perms, perm_ids):
+                    cf_rows.append([add(render_chat(tok, build_prompt(probe, q, perm, self.system)), pids)
+                                    for probe in self.cf_probes])
+            for si, st in enumerate(state_texts):
+                real_rows = [add(render_chat(tok, build_prompt(st, q, perm, self.system)), pids)
+                             for perm, pids in zip(perms, perm_ids)]
+                plan[(si, qi)] = (perms, real_rows, cf_rows)
+
+        # 2. one backend call
+        prompts = [None] * len(prompt_index)
+        for text, i in prompt_index.items():
+            prompts[i] = text
+        logprobs = self.backend.next_token_logprobs(prompts, prompt_ids)
+
+        # 3. per question: raw position-space distributions, priors
+        out: Dict[tuple, Decision] = {}
+        for qi, q in enumerate(questions):
+            perms, _, cf_rows = plan[(0, qi)]
+            p_pos_raw_all = []
+            for si in range(len(states)):
+                lp_real = np.stack([logprobs[r] for r in plan[(si, qi)][1]])   # [P, K]
+                p_pos_raw_all.append((lp_real, np.stack([_softmax(lp) for lp in lp_real])))
+
+            cf_prior = None
+            if want_cf:
+                if cf_rows:
+                    self._cf_cache[q.key] = np.stack([
+                        content_free_prior(np.stack([_softmax(logprobs[r]) for r in rows]))
+                        for rows in cf_rows])                                      # [P, K]
+                cf_prior = self._cf_cache[q.key]
+            prior_used = b_prior = None
+            if level != "raw":
+                stack = np.stack([p for _, p in p_pos_raw_all])                    # [N, P, K]
+                s, n = self._running.get(q.key, (np.zeros(stack.shape[1:]), 0))
+                self._running[q.key] = (s + stack.sum(axis=0), n + len(stack))
+                b_prior = self.running_prior(q)
+                if self.prior == "batch":
+                    prior_used = b_prior
+                elif self.prior == "content_free":
+                    prior_used = cf_prior
+
+            # 4. assemble per state
+            for si, (lp_real, p_pos_raw) in enumerate(p_pos_raw_all):
+                answer_mass = float(np.exp(lp_real).sum(axis=1).mean())
+                raw_probs = marginalize(p_pos_raw[:1], perms[:1])
+                diag: Dict[str, Any] = {"answer_mass": answer_mass, "raw_probs": raw_probs,
+                                        "permutations": len(perms), "perms": perms,
+                                        "p_pos_raw": p_pos_raw}
+                if level == "raw":
+                    probs, achieved = raw_probs, "raw"
+                else:
+                    p_pos = apply_contextual(p_pos_raw, prior_used) if prior_used is not None else p_pos_raw
+                    probs = marginalize(p_pos, perms, self.combine)
+                    achieved = "L0"
+                    diag.update({
+                        "prior_method": self.prior if prior_used is not None else "none",
+                        "prior": prior_used,
+                        "cf_prior": cf_prior,
+                        "batch_prior": b_prior,
+                        "order_flip_raw": flip_rate_across_perms(p_pos_raw, perms),
+                        "order_flip_l0": flip_rate_across_perms(p_pos, perms),
+                        "l0_probs": probs,
+                    })
+                    if level == "L1":
+                        scaler = self._artifacts.get(q.key)
+                        if scaler is None:
+                            raise ValueError(f"no L1 artifact for question {q.id}; call calibrate() first")
+                        probs = scaler.apply(probs)
+                        achieved = "L1"
+                        diag["temperature"] = scaler.temperature
+                out[(si, qi)] = Decision(q, np.asarray(probs), achieved, diag)
+        return out