ruling 0.2.0__py3-none-any.whl

ruling/hosted.py

@@ -0,0 +1,394 @@
+"""Models served behind an OpenAI-compatible chat endpoint.
+
+Questions are rendered exactly as the local decoder engine renders them, and
+the answer is read from the first token's logprobs, restricted to the option
+letters, so a hosted model and a local one are directly comparable. Each
+question is asked once per option ordering, all calls for a record in parallel.
+
+`HostedEngine` speaks to any server that implements `/chat/completions` with
+`top_logprobs`: vLLM, SGLang, llama.cpp, LM Studio, mlx_lm.server, or a
+commercial API. `OpenRouterEngine` adds what only OpenRouter offers, provider
+pinning and per-response cost, so a run can stop at a budget.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+
+import httpx
+import numpy as np
+
+from ruling.calibration import Calibration, Provenance
+from ruling.engine import InputTooLong, RawScore, Scored, TwoStageChoice, answer
+from ruling.prompt import SYSTEM_PROMPT, hosted_user_message, render_question
+from ruling.evals import answer_distribution
+from ruling.questions import (MAX_CHOICE_OPTIONS, MAX_QUESTIONS, Choice, InvalidOptions, Noul, Score, State, SystemOneRequest,
+                              SystemOneResponse, Usage, option_keys)
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+OPENROUTER_PREFIX = "openrouter:"
+HOSTED_PREFIX = "openai:"
+LETTERS = [chr(ord("A") + i) for i in range(20)]
+TOP_LOGPROBS = 20
+"""How many top logprobs a host will return, and so how many options one pass can read.
+
+Hosts differ: OpenAI and OpenRouter allow 20, `mlx_lm.server` allows 11, vLLM is
+configurable. A choice with more options than that is narrowed in two stages, so
+every letter a pass reads can appear in what the host returns.
+"""
+
+
+class HostUnavailable(RuntimeError):
+    """The host is up but not answering right now: overloaded, rate limited, or timing out.
+
+    Distinct from a refusal or a bug so a wrapper can fall back to another engine
+    instead of failing the request.
+    """
+
+
+UNAVAILABLE_STATUSES = (429, 502, 503, 504)
+
+
+def unavailable_or_error(model_id: str, response: httpx.Response) -> RuntimeError:
+    """The exception for a non-200 host response."""
+    text = response.text[:500]
+    if response.status_code in UNAVAILABLE_STATUSES:
+        return HostUnavailable(f"{model_id} is unavailable, returned {response.status_code}: {text}")
+    return RuntimeError(f"{model_id} returned {response.status_code}: {text}")
+
+
+class BudgetExceeded(RuntimeError):
+    pass
+
+
+@dataclass
+class Budget:
+    limit_usd: float
+    spent_usd: float = 0.0
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def charge(self, usd: float) -> None:
+        with self._lock:
+            self.spent_usd += usd
+
+    def check(self) -> None:
+        if self.spent_usd > self.limit_usd:
+            raise BudgetExceeded(f"spent ${self.spent_usd:.4f} of a ${self.limit_usd:.2f} budget")
+
+
+def letter_log_probs(top_logprobs: list[dict], letters: list[str]) -> list[float]:
+    """Normalized log-probabilities over `letters` from one token's top logprobs.
+
+    Spacing variants such as " A" count as "A". A letter absent from the top
+    list gets half the probability of the least likely finite token returned, an
+    upper bound on its true mass, before renormalizing. All arithmetic stays in
+    log space because hosts report logprobs as low as -9999.
+    """
+    finite = [e for e in top_logprobs if math.isfinite(e["logprob"])]
+    grouped: dict[str, list[float]] = {letter: [] for letter in letters}
+    for entry in finite:
+        token = entry["token"].strip()
+        if token in grouped:
+            grouped[token].append(entry["logprob"])
+    if not any(grouped.values()):
+        raise ValueError("none of the answer letters appear in the top logprobs")
+    floor = min(e["logprob"] for e in finite) - math.log(2)
+    values = [_logsumexp(grouped[letter]) if grouped[letter] else floor for letter in letters]
+    total = _logsumexp(values)
+    return [v - total for v in values]
+
+
+def _logsumexp(values: list[float]) -> float:
+    top = max(values)
+    return top + math.log(sum(math.exp(v - top) for v in values))
+
+
+def first_token_top_logprobs(data: dict) -> list[dict] | None:
+    """The first generated token's top logprobs, or None when the host returned none."""
+    logprobs = data["choices"][0].get("logprobs")
+    content = (logprobs or {}).get("content") or []
+    return content[0]["top_logprobs"] if content else None
+
+
+MISSING_LOGPROB_ATTEMPTS = 3
+"""Hosts that advertise logprobs occasionally answer without them under load; the request is repeated this many times."""
+
+
+class HostedEngine(TwoStageChoice):
+    """Scores typed questions with a model behind an OpenAI-compatible endpoint.
+
+    Same interface as the local engines, so the server, the evaluation harness
+    and the calibration fitter cannot tell the difference.
+    """
+
+    prior_debias = False
+    prefix = HOSTED_PREFIX
+
+    def __init__(self, model: str, calibration: Calibration, rotations: int, base_url: str,
+                 api_key: str | None = None, concurrency: int = 16, top_logprobs: int = TOP_LOGPROBS,
+                 extra_body: dict | None = None):
+        self.remote = model.removeprefix(self.prefix)
+        self.model_id = self.prefix + self.remote
+        self.base_url = base_url.rstrip("/")
+        self.revision = None
+        self.adapter = None
+        self.rotations = rotations
+        self.top_logprobs = min(top_logprobs, len(LETTERS))
+        # Hosts disable reasoning differently: vLLM and mlx_lm.server take
+        # chat_template_kwargs, OpenRouter takes `reasoning`. Nothing is sent by
+        # default, because OpenAI rejects body fields it does not know.
+        self.extra_body = extra_body or {}
+        self.pass_options = self.top_logprobs
+        self.max_options = MAX_CHOICE_OPTIONS
+        # Images go to the host as content parts; a host whose model reads only text refuses them itself.
+        self.reads_images = True
+        self.concurrency = concurrency
+        # A hosted model's context is the host's business; ruling does not re-impose one.
+        self.max_input_tokens = 1_000_000
+        self.max_branch_tokens = self.max_input_tokens
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._client = httpx.Client(headers=headers, timeout=120)
+        self._pool = ThreadPoolExecutor(max_workers=concurrency)
+        calibration.check(self.provenance)
+        self.calibration = calibration
+
+    @property
+    def provenance(self) -> Provenance:
+        return Provenance(self.model_id, self.revision, self.rotations, self.prior_debias, self.adapter)
+
+    def body(self, messages: list[dict], max_tokens: int) -> dict:
+        return {"model": self.remote, "messages": messages, "max_tokens": max_tokens,
+                "temperature": 0} | self.extra_body
+
+    def charge(self, data: dict) -> None:
+        """Hook for hosts that report a per-response price."""
+
+    def complete(self, messages: list[dict], max_tokens: int, logprobs: bool) -> dict:
+        body = self.body(messages, max_tokens)
+        if logprobs:
+            body |= {"logprobs": True, "top_logprobs": self.top_logprobs}
+        try:
+            response = self._client.post(f"{self.base_url}/chat/completions", json=body)
+        except httpx.TransportError as exc:
+            raise HostUnavailable(f"{self.model_id} is unreachable at {self.base_url}: {exc}") from exc
+        if response.status_code != 200:
+            raise unavailable_or_error(f"{self.base_url} ({self.remote})", response)
+        data = response.json()
+        self.charge(data)
+        return data
+
+    def _ask(self, state: State, question: Choice | Score | Noul, rotation: int) -> tuple[list[str], list[float], int]:
+        rendered = render_question(question, LETTERS, rotation)
+        messages = [{"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": hosted_user_message(state, rendered)}]
+        for _ in range(MISSING_LOGPROB_ATTEMPTS):
+            data = self.complete(messages, max_tokens=1, logprobs=True)
+            top = first_token_top_logprobs(data)
+            if top is not None:
+                return rendered.keys, letter_log_probs(top, rendered.codes), int(data["usage"]["prompt_tokens"])
+        raise RuntimeError(f"{self.remote} returned no logprobs in {MISSING_LOGPROB_ATTEMPTS} attempts (last host {data.get('provider')})")
+
+    def _score_plain(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        jobs = [(qid, rotation) for qid, question in questions.items()
+                for rotation in range(2 if isinstance(question, Score) else len(option_keys(question)))
+                if rotation < self.rotations]
+        results = list(self._pool.map(lambda job: (job[0], self._ask(state, questions[job[0]], job[1])), jobs))
+        per_question: dict[str, dict[str, list[float]]] = {qid: {} for qid in questions}
+        tokens = 0
+        for qid, (keys, log_probs, prompt_tokens) in results:
+            tokens += prompt_tokens
+            for key, value in zip(keys, log_probs):
+                per_question[qid].setdefault(key, []).append(value)
+        raw = {qid: RawScore(keys=option_keys(q), logits=np.array([np.mean(per_question[qid][k]) for k in option_keys(q)]))
+               for qid, q in questions.items()}
+        return Scored(raw=raw, input_tokens=tokens)
+
+    def evaluate(self, request: SystemOneRequest) -> SystemOneResponse:
+        scored = self.score(request.state, request.questions)
+        answers = {qid: answer(request.questions[qid], scored.raw[qid], self.calibration) for qid in request.questions}
+        return SystemOneResponse(model=self.model_id, answers=answers, usage=Usage(input_tokens=scored.input_tokens))
+
+    def answer(self, question, raw: RawScore):
+        return answer(question, raw, self.calibration)
+
+    def write(self, prompts: list[list[dict]], max_tokens: int) -> list[str]:
+        """Plain text completions for several prompts in parallel, used to write questions."""
+        return list(self._pool.map(lambda messages: self.complete(messages, max_tokens, logprobs=False)["choices"][0]["message"]["content"] or "",
+                                   prompts))
+
+
+class OpenRouterEngine(HostedEngine):
+    """OpenRouter, which pins the serving provider and prices every response."""
+
+    prefix = OPENROUTER_PREFIX
+
+    def __init__(self, model: str, calibration: Calibration, rotations: int, providers: list[str],
+                 budget: Budget, concurrency: int = 16, api_key: str | None = None):
+        self.providers = providers
+        self.budget = budget
+        super().__init__(model, calibration, rotations, OPENROUTER_URL.removesuffix("/chat/completions"),
+                         api_key or os.environ["OPENROUTER_API_KEY"], concurrency)
+
+    def body(self, messages: list[dict], max_tokens: int) -> dict:
+        self.budget.check()
+        return super().body(messages, max_tokens) | {
+            "reasoning": {"enabled": False}, "usage": {"include": True},
+            "provider": {"only": self.providers, "require_parameters": True},
+        }
+
+    def charge(self, data: dict) -> None:
+        self.budget.charge(float(data["usage"]["cost"]))
+
+
+TYPESAFE_PREFIX = "typesafe:"
+
+
+def _error_of(response: httpx.Response) -> dict:
+    """The `error` object of a hosted-shape error body, or an empty dict for any other body."""
+    try:
+        error = response.json().get("error")
+    except ValueError:
+        return {}
+    return error if isinstance(error, dict) else {}
+
+
+def _is_context_rejection(response: httpx.Response) -> bool:
+    """A host that limits context per question says 422 `context_too_long` rather than 413."""
+    return response.status_code == 422 and _error_of(response).get("code") == "context_too_long"
+
+
+def _question_at_fault(response: httpx.Response) -> str | None:
+    param = _error_of(response).get("param")
+    return param.removeprefix("questions.") if isinstance(param, str) and param.startswith("questions.") else None
+
+
+class TypeSafeEngine:
+    """TypeSafe's hosted API, or any server that speaks its protocol, as an engine.
+
+    The host returns probabilities, so the logits handed back are their logs and
+    the host's own calibration is what you get. Useful for scoring a dataset
+    against the hosted model with the same harness, and as the second stage of
+    a cascade, where a local model answers what it is sure of and only the
+    doubtful questions are paid for.
+    """
+
+    prior_debias = False
+    max_options = MAX_CHOICE_OPTIONS
+    # The state goes to the service unchanged, images included; the service decides what it reads.
+    reads_images = True
+    prefix = TYPESAFE_PREFIX
+    rotations = 1
+    """The host orders options as it sees fit; ruling does not re-ask it."""
+
+    def __init__(self, model: str, calibration: Calibration, base_url: str, api_key: str | None,
+                 max_input_tokens: int, max_branch_tokens: int, client: httpx.Client | None = None):
+        self.remote = model.removeprefix(self.prefix)
+        self.model_id = self.prefix + self.remote
+        self.revision = None
+        self.adapter = None
+        self.max_input_tokens = max_input_tokens
+        self.max_branch_tokens = max_branch_tokens
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._client = client or httpx.Client(base_url=base_url.rstrip("/"), headers=headers, timeout=60)
+        calibration.check(self.provenance)
+        self.calibration = calibration
+
+    @property
+    def provenance(self) -> Provenance:
+        return Provenance(self.model_id, self.revision, self.rotations, self.prior_debias, self.adapter)
+
+    def refusal(self, response: httpx.Response) -> RuntimeError:
+        """The exception for a non-200 answer other than the host's context limit."""
+        return unavailable_or_error(self.model_id, response)
+
+    def _check_width(self, questions: dict[str, Choice | Score | Noul]) -> None:
+        for qid, question in questions.items():
+            if len(option_keys(question)) > self.max_options:
+                raise InvalidOptions(qid, f"a choice needs 2 to {self.max_options} options on {self.model_id}, "
+                                          f"got {len(option_keys(question))}")
+
+    def score(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        self._check_width(questions)
+        body = {"state": state, "questions": {qid: q.model_dump(exclude_none=True) for qid, q in questions.items()}}
+        if self.remote:
+            body["model"] = self.remote
+        try:
+            response = self._client.post("/v1/systemone", json=body)
+        except httpx.TransportError as exc:
+            raise HostUnavailable(f"{self.model_id} is unreachable: {exc}") from exc
+        if response.status_code == 413 or _is_context_rejection(response):
+            # The host's own context limit; surfacing it as InputTooLong keeps it a capacity
+            # rejection here too, not a 500.
+            raise InputTooLong(f"{self.model_id} refused the request, over the maximum context length: "
+                               f"{response.text[:300]}", question_id=_question_at_fault(response))
+        if response.status_code != 200:
+            raise self.refusal(response)
+        data = response.json()
+        raw = {}
+        for qid, question in questions.items():
+            keys = option_keys(question)
+            distribution = answer_distribution(data["answers"][qid], keys)
+            # Hosts publish rounded probabilities, zeros included; a zero is a -inf logit
+            # downstream, which softmax and the two-stage Choice already handle, so the
+            # host's distribution passes through exactly instead of picking up a floor.
+            with np.errstate(divide="ignore"):
+                raw[qid] = RawScore(keys=keys, logits=np.log(np.array([distribution[k] for k in keys], dtype=np.float64)))
+        return Scored(raw=raw, input_tokens=int((data.get("usage") or {}).get("input_tokens") or 0))
+
+    def evaluate(self, request: SystemOneRequest) -> SystemOneResponse:
+        scored = self.score(request.state, request.questions)
+        answers = {qid: answer(request.questions[qid], scored.raw[qid], self.calibration) for qid in request.questions}
+        return SystemOneResponse(model=self.model_id, answers=answers, usage=Usage(input_tokens=scored.input_tokens))
+
+    def answer(self, question, raw: RawScore):
+        return answer(question, raw, self.calibration)
+
+
+VLLM_PREFIX = "vllm:"
+VLLM_PASS_OPTIONS = 26
+"""vLLM labels a choice's options A to Z in the prompt, so it reads at most 26 in one pass; a wider
+choice runs in two stages. Its noul and score limits (two sides, ten levels) are within ruling's."""
+VLLM_MAX_QUESTIONS = MAX_QUESTIONS
+"""vLLM takes as many questions in one request as the hosted API; a wider stage one is split across requests."""
+
+
+class VllmEngine(TwoStageChoice, TypeSafeEngine):
+    """A model, or a LoRA adapter by name, behind vLLM's own `/v1/systemone`.
+
+    vLLM reads the first reply token's label logprobs the way ruling does, so
+    the same questions come back in the same shape, batched across requests on
+    the GPU, and an adapter trained here and exported with `ruling export-adapter`
+    answers under the name vLLM was started with. Only the wire differs from the
+    TypeSafe engine: a `vllm:` id, no API key by default, and vLLM's refusals,
+    which are 400s with a reason, surface as invalid input rather than host
+    failures.
+    """
+
+    pass_options = VLLM_PASS_OPTIONS
+    # Whether vLLM's decision endpoint reads images is unverified; refuse rather than send what it may drop.
+    reads_images = False
+    prefix = VLLM_PREFIX
+
+    def score(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        self._check_width(questions)
+        return TwoStageChoice.score(self, state, questions)
+
+    def _score_plain(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        """One request per `VLLM_MAX_QUESTIONS` questions: the first stage of a wide choice asks one per option."""
+        items = list(questions.items())
+        parts = [TypeSafeEngine.score(self, state, dict(items[start:start + VLLM_MAX_QUESTIONS]))
+                 for start in range(0, len(items), VLLM_MAX_QUESTIONS)]
+        return Scored(raw={qid: raw for part in parts for qid, raw in part.raw.items()},
+                      input_tokens=sum(part.input_tokens for part in parts))
+
+    def refusal(self, response: httpx.Response) -> RuntimeError:
+        if response.status_code == 400:
+            return ValueError(f"{self.model_id} refused the request: {response.text[:300]}")
+        if response.status_code == 501:
+            return RuntimeError(f"{self.model_id} has no read strategy in this vLLM; it cannot answer typed questions: "
+                                f"{response.text[:300]}")
+        return unavailable_or_error(self.model_id, response)

ruling/ledger.py

@@ -0,0 +1,120 @@
+"""Every decision a server answers, and the outcomes reported for it later, kept on disk.
+
+An outcome is what turned out to be right for a question, reported by whoever
+learns it: the ticket's final team, whether the refund was granted. Decisions
+with outcomes are training records, so a model can be retrained on its own
+traffic with `ruling export-decisions` and `ruling train`.
+
+SQLite keeps the ledger on disk and indexed, so lookups and memory stay flat
+however much traffic it holds. A retention bounds the disk: a decision nobody
+reports an outcome for within it is dropped, state and all, the next time a
+decision is recorded; a decision with an outcome is training data and stays.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import time
+import uuid
+from collections.abc import Callable, Iterator
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from ruling.evals import Label, Record
+from ruling.questions import SystemOneRequest, SystemOneResponse
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS decisions (
+    id TEXT PRIMARY KEY,
+    created REAL NOT NULL,
+    model TEXT NOT NULL,
+    request TEXT NOT NULL,
+    response TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS decisions_created ON decisions(created);
+CREATE TABLE IF NOT EXISTS outcomes (
+    decision TEXT NOT NULL REFERENCES decisions(id),
+    question TEXT NOT NULL,
+    label TEXT NOT NULL,
+    reported REAL NOT NULL,
+    PRIMARY KEY (decision, question)
+);
+"""
+
+
+class Outcome(BaseModel):
+    """What turned out to be right, per question, in the label forms a training record takes."""
+
+    labels: dict[str, Label] = Field(min_length=1)
+
+
+class UnknownDecision(KeyError):
+    def __str__(self) -> str:
+        return f"no decision {self.args[0]!r} in the ledger"
+
+
+class DecisionLedger:
+    def __init__(self, path: Path, retain_seconds: float | None = None, now: Callable[[], float] = time.time):
+        # Server requests arrive on a thread pool; one connection behind a lock serves them all.
+        self._db = sqlite3.connect(path, check_same_thread=False)
+        self._db.executescript(SCHEMA)
+        self._lock = threading.Lock()
+        self._retain = retain_seconds
+        self._now = now
+
+    def record(self, request: SystemOneRequest, response: SystemOneResponse) -> str:
+        """Store one answered request and return the id outcomes will refer to.
+
+        With a retention, unresolved decisions older than it go first, so the ledger never
+        outgrows the traffic of one retention period plus what has been resolved."""
+        decision = uuid.uuid4().hex
+        now = self._now()
+        with self._lock, self._db:
+            if self._retain is not None:
+                self._db.execute("DELETE FROM decisions WHERE created < ? AND id NOT IN (SELECT decision FROM outcomes)",
+                                 (now - self._retain,))
+            self._db.execute("INSERT INTO decisions VALUES (?, ?, ?, ?, ?)",
+                             (decision, now, response.model, request.model_dump_json(exclude_none=True),
+                              response.model_dump_json(exclude_none=True)))
+        return decision
+
+    def ids(self) -> list[str]:
+        """Every decision still held, oldest first."""
+        with self._lock:
+            return [row[0] for row in self._db.execute("SELECT id FROM decisions ORDER BY created, rowid")]
+
+    def outcome(self, decision: str, labels: dict) -> Record:
+        """Attach outcomes to a decision; a later outcome for the same question replaces the earlier one.
+
+        The labels are checked as a training record would check them, so nothing
+        stored here can fail later in `ruling train`. Returns the record as it now stands.
+        """
+        with self._lock, self._db:
+            row = self._db.execute("SELECT request FROM decisions WHERE id = ?", (decision,)).fetchone()
+            if row is None:
+                raise UnknownDecision(decision)
+            record = self._record(decision, json.loads(row[0]), self._labels(decision) | labels)
+            self._db.executemany("INSERT OR REPLACE INTO outcomes VALUES (?, ?, ?, ?)",
+                                 [(decision, qid, json.dumps(label), self._now()) for qid, label in labels.items()])
+        return record
+
+    def records(self) -> Iterator[Record]:
+        """Every decision with at least one outcome, oldest first, as a training record."""
+        with self._lock:
+            rows = self._db.execute(
+                "SELECT id, request FROM decisions WHERE id IN (SELECT decision FROM outcomes) ORDER BY created"
+            ).fetchall()
+            labeled = [(decision, json.loads(request), self._labels(decision)) for decision, request in rows]
+        for decision, request, labels in labeled:
+            yield self._record(decision, request, labels)
+
+    def _labels(self, decision: str) -> dict:
+        return {qid: json.loads(label) for qid, label in
+                self._db.execute("SELECT question, label FROM outcomes WHERE decision = ?", (decision,))}
+
+    @staticmethod
+    def _record(decision: str, request: dict, labels: dict) -> Record:
+        return Record(id=decision, state=request["state"], questions=request["questions"], labels=labels)

ruling/metrics.py

@@ -0,0 +1,66 @@
+"""Decision metrics in the form hosted decision platforms report, so a ruling report reads like theirs.
+
+Inputs are parallel lists of probability vectors and target vectors over the
+same keys; a target may be one-hot or a soft distribution, whose most likely
+option counts as the correct answer for accuracy.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+THRESHOLDS = (0.5, 0.7, 0.9, 0.95)
+"""Top-1 confidence thresholds for selective accuracy. Reported under their string form; the names never change."""
+
+PROBABILITY_FLOOR = 1e-12
+"""Log loss is taken on probabilities floored here, so an impossible answer costs a finite amount."""
+
+
+def decision_metrics(probs: list[np.ndarray], targets: list[np.ndarray], bins: int = 10) -> dict:
+    """Pooled metrics over paired distributions and targets; `{"n": 0}` for empty input.
+
+    - accuracy: argmax(p) equals argmax(y)
+    - nll: −Σ y log p, averaged
+    - brier: Σ (p − y)², averaged
+    - ece: expected calibration error over `bins` equal-width bins on top-1 confidence
+    - mean_confidence: average top-1 probability
+    - selective: coverage and accuracy among decisions whose top-1 confidence is at least each threshold
+    """
+    if len(probs) != len(targets):
+        raise ValueError("probs and targets must have the same length")
+    n = len(probs)
+    if n == 0:
+        return {"n": 0}
+    confidence, correct, nll, brier = (np.empty(n) for _ in range(4))
+    for i, (p, y) in enumerate(zip(probs, targets)):
+        p, y = np.asarray(p, dtype=np.float64), np.asarray(y, dtype=np.float64)
+        if p.shape != y.shape:
+            raise ValueError(f"decision {i}: probabilities have {p.size} entries, target has {y.size}")
+        predicted = int(p.argmax())
+        confidence[i] = p[predicted]
+        correct[i] = predicted == int(y.argmax())
+        nll[i] = -(y * np.log(np.maximum(p, PROBABILITY_FLOOR))).sum()
+        brier[i] = ((p - y) ** 2).sum()
+    ece = 0.0
+    for b in range(bins):
+        lo, hi = b / bins, (b + 1) / bins
+        # A confidence of exactly zero belongs to the first bin rather than to none.
+        member = ((lo < confidence) & (confidence <= hi)) | ((b == 0) & (confidence == 0))
+        if member.any():
+            ece += member.mean() * abs(confidence[member].mean() - correct[member].mean())
+    selective = {}
+    for threshold in THRESHOLDS:
+        covered = confidence >= threshold
+        selective[str(threshold)] = {
+            "coverage": float(covered.mean()),
+            "accuracy": float(correct[covered].mean()) if covered.any() else None,
+        }
+    return {
+        "n": n,
+        "accuracy": float(correct.mean()),
+        "nll": float(nll.mean()),
+        "brier": float(brier.mean()),
+        "ece": float(ece),
+        "mean_confidence": float(confidence.mean()),
+        "selective": selective,
+    }