ruling 0.2.0__py3-none-any.whl

ruling/evals.py

@@ -0,0 +1,459 @@
+"""Labeled-dataset evaluation and temperature fitting.
+
+A dataset is JSONL, or a JSON array. Each record holds a state, a questions
+map in the API shape, and optionally: `labels`, gold answers by question id
+(an option key for Choice, a level index or level text for Score, a boolean
+for Noul); `references`, published answer distributions by question id and
+source name, such as Jev's own answers, used to report agreement; `family`
+for grouped balanced-accuracy reporting; and an optional `id` and `weight`.
+
+Rows written for hosted decision platforms are the same shape and load unchanged. Their extra label forms
+are normalized on validation: the strings "true"/"false" become a boolean, a
+score option id such as "2" becomes the index 2, and a soft label (a
+probability of yes, a per-option map, or a per-level list) becomes the
+`target` reference with the label set to its most likely option. A
+single-question row holds `question` and `label` (or `target`, or
+`expected`) instead of maps, and becomes one question named `decision`.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections import defaultdict
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+from pydantic import BaseModel, Field, model_validator
+
+from ruling.calibration import (
+    Calibration,
+    Provenance,
+    choice_confidence,
+    confident_error_rate,
+    coverage_at_error,
+    expected_calibration_error,
+    fit_temperature,
+    softmax,
+)
+from ruling.engine import InputTooLong
+from ruling.metrics import decision_metrics
+from ruling.questions import (
+    MAX_CHOICE_OPTIONS,
+    Choice,
+    InvalidOptions,
+    Noul,
+    Question,
+    Score,
+    State,
+    SystemOneRequest,
+    UnsupportedModality,
+    check_askable,
+    check_images,
+    check_readable,
+    option_keys,
+)
+
+Distribution = dict[str, float]
+Label = str | int | bool | float | list[float] | dict[str, float]
+
+TARGET = "target"
+"""The reference name whose distribution is the soft training and scoring target for a question."""
+
+SINGLE_QUESTION = "decision"
+"""The question id a single-question row is given, the name hosted platforms use for it."""
+
+SOFT_SUM_TOLERANCE = 0.02
+"""How far a soft label may stray from summing to one before it is refused; the tolerance hosted platforms allow."""
+
+NOUL_KEYS = {"true": "yes", "false": "no"}
+"""The true/false noul option ids hosted platforms use, mapped onto ruling's canonical keys."""
+
+
+ERROR_BUDGET = 0.05
+"""The error rate a deployment accepts among automated decisions; coverage is reported against it."""
+
+
+class Record(BaseModel):
+    id: str | int | None = None
+    state: State
+    questions: dict[str, Question] = Field(min_length=1)
+    labels: dict[str, Label] = Field(default_factory=dict)
+    references: dict[str, dict[str, Distribution]] = Field(default_factory=dict)
+    family: str | None = None
+    weight: float = Field(default=1.0, ge=0)
+    """How much the record counts in a training update."""
+
+    @model_validator(mode="after")
+    def labels_and_references_match_questions(self) -> "Record":
+        check_askable(self.questions, MAX_CHOICE_OPTIONS)
+        check_images(self.state)
+        unknown = (set(self.labels) | set(self.references)) - set(self.questions)
+        if unknown:
+            raise ValueError(f"labels or references for unknown questions {sorted(unknown)}")
+        for qid, label in self.labels.items():
+            question = self.questions[qid]
+            try:
+                soft = soft_target(question, label)
+                if soft is not None:
+                    self.references.setdefault(qid, {})[TARGET] = soft
+                    label = label_for_key(question, max(option_keys(question), key=soft.__getitem__))
+                self.labels[qid] = canonical_label(question, label)
+                label_index(question, self.labels[qid])
+            except ValueError as error:
+                raise ValueError(f"label for {qid!r}: {error}") from error
+        for qid, sources in self.references.items():
+            keys = set(option_keys(self.questions[qid]))
+            for name, distribution in sources.items():
+                if set(distribution) != keys:
+                    raise ValueError(f"reference {name!r} for {qid!r} must cover exactly {sorted(keys)}")
+        return self
+
+
+@dataclass(frozen=True)
+class Observation:
+    record: int
+    question_id: str
+    question_type: str
+    family: str | None
+    keys: list[str]
+    logits: np.ndarray | None
+    """None when the record exceeded the engine's input limit and could not be scored."""
+    label: int | None
+    references: dict[str, np.ndarray]
+    reversed_argmax: int | None
+    latency_ms: float | None = None
+    """Wall time for the whole record's request, on every observation the request produced."""
+
+
+def load_dataset(path: Path) -> list[Record]:
+    """Records from a `.jsonl` file (one row per line) or a `.json` array, in ruling's row shape or a hosted platform's."""
+    text = path.read_text()
+    if path.suffix == ".json":
+        rows = json.loads(text)
+        if not isinstance(rows, list):
+            raise ValueError(f"{path}: a .json dataset must be an array of rows")
+    else:
+        rows = [json.loads(line) for line in text.splitlines() if line.strip()]
+    records = []
+    for index, row in enumerate(rows, start=1):
+        try:
+            records.append(Record.model_validate(as_record(row)))
+        except ValueError as error:
+            where = f"row {index}" + (f" ({row['id']})" if isinstance(row, dict) and row.get("id") is not None else "")
+            raise ValueError(f"{path} {where}: {error}") from error
+    return records
+
+
+def as_record(row: dict) -> dict:
+    """A row in any accepted shape as a record dict: a single-question row becomes one question, `decision`."""
+    if "questions" in row:
+        return row
+    if "question" not in row:
+        raise ValueError("a row needs a 'questions' map or a single 'question'")
+    label = _single_label(row)
+    record = {k: v for k, v in row.items() if k not in ("question", "label", "target", "expected")}
+    record["questions"] = {SINGLE_QUESTION: row["question"]}
+    if label is not None:
+        record["labels"] = {SINGLE_QUESTION: label}
+    return record
+
+
+def _single_label(row: dict):
+    """Precedence for a single-question row: `expected`, else a soft `target`, else `label`, else `target`."""
+    if "expected" in row:
+        return row["expected"]
+    target, label = row.get("target"), row.get("label")
+    if isinstance(target, (list, float, dict)) and not isinstance(target, bool):
+        return target
+    return label if label is not None else target
+
+
+def soft_target(question: Choice | Score | Noul, label: Label) -> Distribution | None:
+    """A soft label as a distribution over the question's canonical keys, or None when the label is hard.
+
+    Noul takes a probability of yes or a map over true/false (or yes/no); Choice a map over its options or a
+    list in option order; Score a map over "0".."n-1" or a list from the lowest level up.
+    """
+    keys = option_keys(question)
+    if isinstance(label, (bool, str)):
+        return None
+    if isinstance(label, dict):
+        rename = NOUL_KEYS if isinstance(question, Noul) else {}
+        probs = {rename.get(str(k), str(k)): float(v) for k, v in label.items()}
+    elif isinstance(label, list):
+        if isinstance(question, Noul):
+            raise ValueError("a noul soft label is a probability of yes or a map over true/false, not a list")
+        if len(label) != len(keys):
+            raise ValueError(f"a soft label needs {len(keys)} probabilities, got {len(label)}")
+        probs = dict(zip(keys, map(float, label)))
+    elif isinstance(question, Noul):
+        if not 0 <= label <= 1:
+            raise ValueError(f"a noul probability must be within [0, 1], got {label!r}")
+        probs = {"yes": float(label), "no": 1.0 - float(label)}
+    else:
+        return None
+    if set(probs) != set(keys):
+        raise ValueError(f"a soft label must cover exactly {keys}, got {sorted(probs)}")
+    if min(probs.values()) < 0:
+        raise ValueError("a soft label must be non-negative")
+    if abs(sum(probs.values()) - 1.0) > SOFT_SUM_TOLERANCE:
+        raise ValueError(f"a soft label must sum to 1, got {sum(probs.values()):.4f}")
+    return probs
+
+
+def canonical_label(question: Choice | Score | Noul, label: Label) -> str | int | bool:
+    """The string forms of a hard label in ruling's form: "true"/"false" to a bool, a score id "2" to 2."""
+    if isinstance(question, Noul) and isinstance(label, str) and label.lower() in ("true", "false", "yes", "no"):
+        return label.lower() in ("true", "yes")
+    if isinstance(question, Score) and isinstance(label, str) and label.isdigit():
+        return int(label)
+    return label
+
+
+def label_for_key(question: Choice | Score | Noul, key: str) -> str | int | bool:
+    """The hard label that names one canonical key."""
+    if isinstance(question, Noul):
+        return key == "yes"
+    if isinstance(question, Score):
+        return int(key)
+    return key
+
+
+def label_index(question: Choice | Score | Noul, label: str | int | bool) -> int:
+    keys = option_keys(question)
+    if isinstance(question, Noul):
+        if not isinstance(label, bool):
+            raise ValueError(f"noul label must be a boolean, got {label!r}")
+        return 0 if label else 1
+    if isinstance(question, Score):
+        if isinstance(label, bool):
+            raise ValueError("score label must be a level index or level text")
+        if isinstance(label, int):
+            if not 0 <= label < len(keys):
+                raise ValueError(f"score level {label} out of range")
+            return label
+        if label in question.criteria:
+            return question.criteria.index(label)
+        raise ValueError(f"unknown score level {label!r}")
+    if label not in keys:
+        raise ValueError(f"unknown choice option {label!r}")
+    return keys.index(label)
+
+
+def answer_distribution(answer: dict, keys: list[str]) -> Distribution:
+    """A published System One answer as a distribution over canonical keys."""
+    if answer["type"] == "noul":
+        return {"yes": float(answer["noul"]), "no": 1.0 - float(answer["noul"])}
+    probabilities = {str(k): float(v) for k, v in answer["probabilities"].items()}
+    missing = set(keys) - set(probabilities)
+    if missing:
+        raise ValueError(f"published answer lacks options {sorted(missing)}")
+    return {key: probabilities[key] for key in keys}
+
+
+def collect(engine, records: list[Record], extra_references: dict[tuple[int, str], Distribution] | None = None) -> list[Observation]:
+    """Score every record. Choice questions are also scored with reversed options.
+
+    A record longer than the engine accepts becomes unscored observations, so a
+    report can say how much of a dataset a model could read at all.
+    """
+    observations = []
+    for index, record in enumerate(records):
+        started = time.perf_counter()
+        try:
+            scored = engine.score(record.state, record.questions)
+        except InputTooLong:
+            observations += [Observation(record=index, question_id=qid, question_type=q.type, family=record.family,
+                                         keys=option_keys(q), logits=None,
+                                         label=label_index(q, record.labels[qid]) if qid in record.labels else None,
+                                         references={}, reversed_argmax=None)
+                             for qid, q in record.questions.items()]
+            continue
+        reversed_questions = {
+            qid: Choice(instructions=q.instructions, criteria=dict(reversed(list(q.criteria.items()))))
+            for qid, q in record.questions.items()
+            if isinstance(q, Choice)
+        }
+        latency_ms = (time.perf_counter() - started) * 1000
+        reversed_scored = engine.score(record.state, reversed_questions) if reversed_questions else None
+        for qid, question in record.questions.items():
+            raw = scored.raw[qid]
+            reversed_argmax = None
+            if qid in reversed_questions:
+                rev = reversed_scored.raw[qid]
+                reversed_argmax = raw.keys.index(rev.keys[int(rev.logits.argmax())])
+            references = dict(record.references.get(qid, {}))
+            if extra_references and (index, qid) in extra_references:
+                references["live"] = extra_references[(index, qid)]
+            observations.append(
+                Observation(
+                    record=index,
+                    question_id=qid,
+                    question_type=question.type,
+                    family=record.family,
+                    keys=raw.keys,
+                    logits=raw.logits,
+                    label=label_index(question, record.labels[qid]) if qid in record.labels else None,
+                    references={name: np.array([d[k] for k in raw.keys]) for name, d in references.items()},
+                    reversed_argmax=reversed_argmax,
+                    latency_ms=latency_ms,
+                )
+            )
+    return observations
+
+
+def balanced_accuracy(predicted: list[int], gold: list[int]) -> float:
+    recalls = []
+    for label in sorted(set(gold)):
+        hits = sum(1 for p, g in zip(predicted, gold) if g == label and p == label)
+        recalls.append(hits / gold.count(label))
+    return float(np.mean(recalls))
+
+
+def total_variation(p: np.ndarray, q: np.ndarray) -> float:
+    return float(0.5 * np.abs(p - q).sum())
+
+
+def report(observations: list[Observation], calibration: Calibration) -> dict:
+    """Per-type summaries under `question_types`, plus the pooled decision metrics, by type and by question."""
+    by_type: dict[str, list[Observation]] = defaultdict(list)
+    for obs in observations:
+        by_type[obs.question_type].append(obs)
+    result = {"question_types": {}, **decision_blocks(observations, calibration)}
+    latencies = sorted({(o.record, o.latency_ms) for o in observations if o.latency_ms is not None})
+    if latencies:
+        values = np.array([ms for _, ms in latencies])
+        result["latency_ms"] = {"p50": float(np.percentile(values, 50)), "p95": float(np.percentile(values, 95)),
+                                "records": len(values)}
+    for question_type, everything in by_type.items():
+        group = [o for o in everything if o.logits is not None]
+        if not group:
+            result["question_types"][question_type] = {"count": 0, "unscored": len(everything)}
+            continue
+        temperature = calibration.temperature(question_type)
+        probs = [softmax(o.logits, temperature) for o in group]
+        summary = {"count": len(group), "unscored": len(everything) - len(group), "temperature": temperature,
+                   "mean_confidence": float(np.mean([choice_confidence(p) for p in probs]))}
+        labeled = [(o, p) for o, p in zip(group, probs) if o.label is not None]
+        summary["labeled_count"] = len(labeled)
+        if labeled:
+            predicted = [int(p.argmax()) for _, p in labeled]
+            gold = [o.label for o, _ in labeled]
+            correct = [a == b for a, b in zip(predicted, gold)]
+            summary.update({
+                "accuracy": float(np.mean(correct)),
+                "balanced_accuracy": balanced_accuracy(predicted, gold),
+                "expected_calibration_error": expected_calibration_error([float(p.max()) for _, p in labeled], correct),
+                "brier": float(np.mean([((p - np.eye(len(p))[g]) ** 2).sum() for (_, p), g in zip(labeled, gold)])),
+                "negative_log_likelihood": float(np.mean([-np.log(max(p[g], 1e-12)) for (_, p), g in zip(labeled, gold)])),
+                "coverage_at_5pct_error": coverage_at_error([float(p.max()) for _, p in labeled], correct, ERROR_BUDGET),
+                "confident_error_rate": confident_error_rate([float(p.max()) for _, p in labeled], correct),
+            })
+            families = defaultdict(list)
+            for (o, _), p, g in zip(labeled, predicted, gold):
+                if o.family is not None:
+                    families[o.family].append((p, g))
+            if families and len(families) == len({o.family for o, _ in labeled}):
+                per_family = {name: balanced_accuracy([p for p, _ in pairs], [g for _, g in pairs])
+                              for name, pairs in families.items()}
+                summary["family_balanced_accuracy"] = per_family
+                summary["mean_family_balanced_accuracy"] = float(np.mean(list(per_family.values())))
+        if question_type == "choice":
+            flips = sum(1 for o, p in zip(group, probs) if o.reversed_argmax != int(p.argmax()))
+            summary["option_reversal_flips"] = flips
+            summary["option_reversal_flip_rate"] = flips / len(group)
+        names = sorted({name for o in group for name in o.references})
+        if names:
+            summary["references"] = {name: _reference_summary(name, group, probs) for name in names}
+        result["question_types"][question_type] = summary
+    return result
+
+
+def decision_blocks(observations: list[Observation], calibration: Calibration) -> dict:
+    """`decision_metrics` on every labeled, scored question: pooled, keyed by type, and keyed by question id.
+
+    The target is the record's soft `target` reference where it has one, else one-hot on the label.
+    """
+    labeled = [o for o in observations if o.logits is not None and o.label is not None]
+    probs = [softmax(o.logits, calibration.temperature(o.question_type)) for o in labeled]
+    targets = [o.references[TARGET] if TARGET in o.references else np.eye(len(o.keys))[o.label] for o in labeled]
+
+    def block(select) -> dict:
+        chosen = [i for i, o in enumerate(labeled) if select(o)]
+        return decision_metrics([probs[i] for i in chosen], [targets[i] for i in chosen])
+
+    return {
+        "overall": block(lambda o: True),
+        "by_type": {t: block(lambda o: o.question_type == t) for t in dict.fromkeys(o.question_type for o in labeled)},
+        "by_question": {q: block(lambda o: o.question_id == q) for q in dict.fromkeys(o.question_id for o in labeled)},
+    }
+
+
+def _reference_summary(name: str, group: list[Observation], probs: list[np.ndarray]) -> dict:
+    pairs = [(o, p, o.references[name]) for o, p in zip(group, probs) if name in o.references]
+    summary = {
+        "count": len(pairs),
+        "argmax_agreement": float(np.mean([int(p.argmax()) == int(r.argmax()) for _, p, r in pairs])),
+        "mean_total_variation": float(np.mean([total_variation(p, r) for _, p, r in pairs])),
+    }
+    labeled = [(o, p, r) for o, p, r in pairs if o.label is not None]
+    if labeled:
+        summary["labeled_count"] = len(labeled)
+        summary["reference_accuracy"] = float(np.mean([int(r.argmax()) == o.label for o, _, r in labeled]))
+        summary["ruling_accuracy_on_same_rows"] = float(np.mean([int(p.argmax()) == o.label for o, p, _ in labeled]))
+    return summary
+
+
+def fit(observations: list[Observation], provenance: Provenance) -> Calibration:
+    by_type: dict[str, list[Observation]] = defaultdict(list)
+    for obs in observations:
+        if obs.label is not None and obs.logits is not None:
+            by_type[obs.question_type].append(obs)
+    if not by_type:
+        raise ValueError("cannot fit a calibration without labeled questions")
+    return Calibration(
+        temperatures={
+            question_type: fit_temperature([o.logits for o in group], [o.label for o in group])
+            for question_type, group in by_type.items()
+        },
+        provenance=provenance,
+    )
+
+
+def write_predictions(observations: list[Observation], calibration: Calibration, path: Path) -> None:
+    """One line per scored question with its calibrated distribution, for ensembling and audits."""
+    with path.open("w") as out:
+        for obs in observations:
+            if obs.logits is None:
+                continue
+            probs = softmax(obs.logits, calibration.temperature(obs.question_type))
+            out.write(json.dumps({"record": obs.record, "question_id": obs.question_id, "type": obs.question_type,
+                                  "keys": obs.keys, "probabilities": [float(p) for p in probs], "label": obs.label}) + "\n")
+
+
+def predict(engine, records: list[Record]) -> Iterator[dict]:
+    """One line per record, in order: its answers as the API returns them, or the API's error for a row it refuses.
+
+    Labels are ignored. A refused row keeps its place, so line i always belongs to record i.
+    """
+    for record in records:
+        line = {"id": record.id} if record.id is not None else {}
+        try:
+            check_askable(record.questions, engine.max_options)
+            check_readable(record.state, engine.reads_images)
+            response = engine.evaluate(SystemOneRequest(state=record.state, questions=record.questions))
+            line["answers"] = {qid: a.model_dump(mode="json") for qid, a in response.answers.items()}
+        except InputTooLong as exc:
+            line["error"] = {"code": "context_too_long", "message": str(exc)}
+        except InvalidOptions as exc:
+            line["error"] = {"code": "invalid_options", "message": str(exc)}
+        except UnsupportedModality as exc:
+            line["error"] = {"code": "unsupported_modality", "message": str(exc)}
+        yield line
+
+
+def write_json(path: Path, data: dict) -> None:
+    path.write_text(json.dumps(data, indent=2) + "\n")

ruling/failover.py

@@ -0,0 +1,62 @@
+"""One engine answers; another takes over whenever the first is unavailable.
+
+Built for a hosted primary such as `typesafe:jev-latest` with a local model
+behind it: a 503, a 429 or a timeout from the host is answered by the fallback
+instead of failing the request. Which engine answered is in the response's
+`model`, so a caller can tell a fallback answer from a primary one.
+
+Only unavailability triggers the fallback. A refusal (413, 400) or a bug is
+raised as it would be from the primary alone.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from ruling.calibration import Provenance
+from ruling.engine import RawScore, Scored
+from ruling.hosted import HostUnavailable
+from ruling.questions import Choice, Noul, Score, State, SystemOneRequest, SystemOneResponse, Usage
+
+PREFIX = "failover:"
+log = logging.getLogger("ruling")
+
+
+class FailoverEngine:
+    """Same interface as the engines it wraps; limits are the primary's, since the fallback is best effort."""
+
+    def __init__(self, primary, fallback):
+        self.primary, self.fallback = primary, fallback
+        self.model_id = f"{PREFIX}{primary.model_id}>{fallback.model_id}"
+        self.revision = primary.revision
+        self.adapter = primary.adapter
+        self.rotations = primary.rotations
+        self.prior_debias = primary.prior_debias
+        self.max_input_tokens = primary.max_input_tokens
+        self.max_branch_tokens = primary.max_branch_tokens
+        self.max_options = primary.max_options
+        self.reads_images = primary.reads_images
+        self.calibration = primary.calibration
+
+    @property
+    def provenance(self) -> Provenance:
+        return Provenance(self.model_id, self.revision, self.rotations, self.prior_debias, self.adapter)
+
+    def _answering(self, state: State, questions: dict[str, Choice | Score | Noul]):
+        """The engine that answered, and its scores."""
+        try:
+            return self.primary, self.primary.score(state, questions)
+        except HostUnavailable as exc:
+            log.warning("failover from %s to %s: %s", self.primary.model_id, self.fallback.model_id, exc)
+            return self.fallback, self.fallback.score(state, questions)
+
+    def score(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        return self._answering(state, questions)[1]
+
+    def evaluate(self, request: SystemOneRequest) -> SystemOneResponse:
+        engine, scored = self._answering(request.state, request.questions)
+        answers = {qid: engine.answer(request.questions[qid], scored.raw[qid]) for qid in request.questions}
+        return SystemOneResponse(model=engine.model_id, answers=answers, usage=Usage(input_tokens=scored.input_tokens))
+
+    def answer(self, question, raw: RawScore):
+        return self.primary.answer(question, raw)

ruling/history.html

@@ -0,0 +1,78 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>ruling retraining</title>
+<style>
+  :root { --bg: #fafaf7; --fg: #1b1b1b; --muted: #6b6b6b; --line: #dcdcd5; --bar: #2f6f4f; --accent: #b3451f; }
+  @media (prefers-color-scheme: dark) { :root { --bg: #161615; --fg: #ecece6; --muted: #9a9a92; --line: #33332f; --bar: #5fae85; --accent: #e0774d; } }
+  body { margin: 0; background: var(--bg); color: var(--fg); font: 15px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; }
+  main { max-width: 960px; margin: 0 auto; padding: 24px 16px; display: grid; gap: 16px; }
+  h1 { margin: 0; font-size: 18px; }
+  h1 small, .meta { color: var(--muted); font-weight: normal; font-size: 12px; }
+  h1 small { margin-left: 8px; }
+  a { color: inherit; }
+  h2 { margin: 8px 0 4px; font-size: 15px; }
+  .scroll { overflow-x: auto; }
+  table { border-collapse: collapse; width: 100%; font-size: 13px; }
+  th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
+  th { color: var(--muted); font-weight: normal; font-size: 12px; }
+  td.num { text-align: right; white-space: nowrap; }
+  .promoted { color: var(--bar); }
+  .kept { color: var(--muted); }
+  .serving { font-weight: bold; }
+  .err { color: var(--accent); }
+</style>
+</head>
+<body>
+<main>
+  <h1>ruling <small>retraining history · <a href="/">playground</a></small></h1>
+  <div id="out" class="meta">loading…</div>
+</main>
+<script>
+const out = document.getElementById("out");
+const pct = x => x == null ? "–" : x.toFixed(3);
+function change(current, candidate, key) {
+  return `${pct(current[key])} → ${pct(candidate[key])}`;
+}
+function text(value) {
+  const span = document.createElement("span");
+  span.textContent = value;
+  return span.innerHTML;
+}
+let key = "";
+try { key = sessionStorage.getItem("ruling-key") || ""; } catch {}
+fetch("/v1/retrain", { headers: key ? { authorization: `Bearer ${key}` } : {} }).then(async r => {
+  const data = await r.json();
+  if (r.status === 401) throw new Error("this server needs its API key; enter it in the playground first");
+  if (!r.ok) throw new Error(`${r.status} ${data.error.code}: ${data.error.message}`);
+  return data;
+}).then(data => {
+  if (!data.models.length) {
+    out.innerHTML = "No served model is a <code>ruling retrain</code> adapter. Serve one with " +
+      "<code>RULING_ADAPTER=&lt;adapters&gt;/current</code>.";
+    return;
+  }
+  let html = "";
+  for (const m of data.models) {
+    html += `<h2>${text(m.name)} <span class="meta">serving ${text(m.current ?? "the base model")}</span></h2>`;
+    if (!m.versions.length) { html += '<div class="meta">no finished runs yet</div>'; continue; }
+    html += '<div class="scroll"><table><thead><tr><th>version</th><th>from</th><th class="num">records</th>' +
+            '<th class="num">held-out accuracy</th><th class="num">held-out log loss</th><th>verdict</th></tr></thead><tbody>';
+    for (const v of [...m.versions].reverse()) {
+      const serving = v.version === m.current ? " serving" : "";
+      html += `<tr class="${v.promoted ? "promoted" : "kept"}${serving}" data-version="${text(v.version)}">` +
+              `<td>${text(v.version)}</td><td>${text(v.started_from ?? "base")}</td>` +
+              `<td class="num">${v.records}</td><td class="num">${change(v.current, v.candidate, "accuracy")}</td>` +
+              `<td class="num">${change(v.current, v.candidate, "nll")}</td>` +
+              `<td>${v.promoted ? "promoted" : "kept"}: ${text(v.reason)}</td></tr>`;
+    }
+    html += "</tbody></table></div>";
+  }
+  out.className = "";
+  out.innerHTML = html;
+}).catch(e => { out.className = "err"; out.textContent = `could not load the history: ${e.message}`; });
+</script>
+</body>
+</html>