ruling 0.2.0__py3-none-any.whl

ruling/__init__.py

@@ -0,0 +1,25 @@
+"""ruling: typed, probabilistic decisions from a local open-weight model."""
+
+from ruling.questions import (
+    Choice,
+    ChoiceAnswer,
+    Noul,
+    NoulAnswer,
+    Score,
+    ScoreAnswer,
+    SystemOneRequest,
+    SystemOneResponse,
+)
+from ruling.sdk import RulingClient
+
+__all__ = [
+    "Choice",
+    "ChoiceAnswer",
+    "Noul",
+    "NoulAnswer",
+    "RulingClient",
+    "Score",
+    "ScoreAnswer",
+    "SystemOneRequest",
+    "SystemOneResponse",
+]

ruling/against.py

@@ -0,0 +1,50 @@
+"""Send a dataset to the hosted TypeSafe API so Jev's live answers can sit next to ruling's.
+
+TypeSafe's HTTP request and response shapes are the same as ruling's, so this
+is a thin client: bearer token, one POST per record, answers converted to
+distributions over the record's canonical keys. Nothing but the record's state
+and questions is sent.
+"""
+
+from __future__ import annotations
+
+import os
+
+import httpx
+
+from ruling.evals import Distribution, Record, answer_distribution
+from ruling.questions import option_keys
+
+API_KEY_ENV = "TYPESAFE_API_KEY"
+BASE_URL_ENV = "TYPESAFE_BASE_URL"
+DEFAULT_BASE_URL = "https://api.typesafe.ai"
+
+
+def live_answers(
+    records: list[Record],
+    base_url: str | None = None,
+    api_key: str | None = None,
+    model: str | None = None,
+    timeout: float = 30.0,
+):
+    """Yield (record index, answer distributions by question id) from the hosted API, one request per record."""
+    base_url = base_url or os.environ.get(BASE_URL_ENV, DEFAULT_BASE_URL)
+    api_key = api_key or os.environ.get(API_KEY_ENV)
+    headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
+    with httpx.Client(base_url=base_url, headers=headers, timeout=timeout) as client:
+        for index, record in enumerate(records):
+            body = {"state": record.state,
+                    "questions": {qid: q.model_dump(exclude_none=True) for qid, q in record.questions.items()}}
+            if model is not None:
+                body["model"] = model
+            response = client.post("/v1/systemone", json=body)
+            response.raise_for_status()
+            yield index, {qid: answer_distribution(answer, option_keys(record.questions[qid]))
+                          for qid, answer in response.json()["answers"].items()}
+
+
+def collect_live(records: list[Record], base_url: str | None = None, api_key: str | None = None,
+                 model: str | None = None, timeout: float = 30.0) -> dict[tuple[int, str], Distribution]:
+    return {(index, qid): distribution
+            for index, answers in live_answers(records, base_url, api_key, model, timeout)
+            for qid, distribution in answers.items()}

ruling/build.py

@@ -0,0 +1,257 @@
+"""Build labeled datasets from pinned public sources.
+
+Nothing here is committed; the builders download a pinned revision and write
+records in the ruling dataset format. WANLI rows follow the 256-row selection
+published by TheoLeeCJ/openjev so results stay comparable. SST-5 gives an
+ordinal five-level Score benchmark. The TypeSafe and Every builders carry
+Jev's own published answers as references, so agreement with Jev can be
+reported on identical inputs.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import random
+import re
+import zipfile
+from pathlib import Path
+
+import httpx
+from huggingface_hub import hf_hub_download
+
+from ruling.evals import Distribution, answer_distribution
+
+DATASETS = Path(__file__).resolve().parents[1] / "datasets"
+
+WANLI = {"repo": "alisawuffles/WANLI", "revision": "61c95318fd71c55b6ba355d76253254615f387ec", "file": "test.jsonl"}
+SST5 = {"repo": "SetFit/sst5", "revision": "e51bdcd8cd3a30da231967c1a249ba59361279a3", "file": "test.jsonl"}
+WANLI_SELECTION = DATASETS / "wanli256-selection.jsonl"
+TYPESAFE_SELECTION = DATASETS / "typesafe102-selection.jsonl"
+TYPESAFE_CASES = "https://evals.typesafe.ai/{workflow}-cases.js"
+EVERY_EXPERIMENTS = "https://typesafe-parallel-judgment-lab.every-4573.chatgpt.site/downloads/experiments.json"
+EVERY_ARCHIVE = "https://typesafe-parallel-judgment-lab.every-4573.chatgpt.site/downloads/typesafe-lab-source.zip"
+EVERY_EXPERIMENTS_GENERATED_AT = "2026-08-28T19:55:23.191Z"
+EVERY_ARCHIVE_SHA256 = "9fbf42e9d9e7cd3b072e0271a0959dbfdca85618e93fe4f0ca519d683c418bf0"
+
+NLI_DESCRIPTIONS = {
+    "supported": "The evidence establishes the claim",
+    "insufficient": "The evidence does not establish either",
+    "contradicted": "The evidence establishes the opposite",
+}
+NLI_LABELS = {"entailment": "supported", "neutral": "insufficient", "contradiction": "contradicted"}
+SST5_LEVELS = ["very negative", "negative", "neutral", "positive", "very positive"]
+
+
+def _rows(source: dict) -> list[dict]:
+    path = hf_hub_download(source["repo"], source["file"], repo_type="dataset", revision=source["revision"])
+    return [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
+
+
+def _fetch(url: str) -> bytes:
+    response = httpx.get(url, timeout=60.0, follow_redirects=True)
+    response.raise_for_status()
+    return response.content
+
+
+def build_wanli() -> list[dict]:
+    """256 NLI rows as a 3-way Choice plus a Noul on entailment."""
+    by_id = {str(row["id"]): row for row in _rows(WANLI)}
+    selection = [json.loads(line) for line in WANLI_SELECTION.read_text().splitlines() if line.strip()]
+    records = []
+    for item in selection:
+        row = by_id[str(item["upstream"]["source_id"])]
+        gold = NLI_LABELS[row["gold"]]
+        records.append({
+            "family": item["family"],
+            "source_id": row["id"],
+            "state": row["premise"],
+            "questions": {
+                "verdict": {"type": "choice",
+                            "instructions": "Assess the claim using only the supplied evidence: " + row["hypothesis"],
+                            "criteria": {key: NLI_DESCRIPTIONS[key] for key in item["option_ids"]}},
+                "supported": {"type": "noul",
+                              "instructions": "The evidence establishes this claim: " + row["hypothesis"]},
+            },
+            "labels": {"verdict": gold, "supported": gold == "supported"},
+        })
+    if len(records) != 256:
+        raise ValueError(f"expected 256 WANLI rows, built {len(records)}")
+    return records
+
+
+def build_sst5(limit: int, seed: int = 0) -> list[dict]:
+    """A seeded sample of SST-5 test sentences as a five-level Score."""
+    rows = _rows(SST5)
+    sample = random.Random(seed).sample(rows, limit)
+    return [{
+        "family": "sentiment",
+        "state": row["text"],
+        "questions": {"sentiment": {"type": "score", "instructions": "What is the sentiment of the review sentence?",
+                                    "criteria": SST5_LEVELS}},
+        "labels": {"sentiment": int(row["label"])},
+    } for row in sample]
+
+
+def build_typesafe() -> list[dict]:
+    """The 102 TypeSafe public-eval rows TheoLeeCJ aligned, with Jev's published answers.
+
+    Labels are the argmax of TypeSafe's reference, the mean of GPT-6 Astra and
+    Claude Fable 5.1 at high reasoning, so they measure agreement with big
+    models, not ground truth. The selection manifest pins a hash of each
+    workflow payload and the build refuses to run if a payload has changed.
+    """
+    selection = [json.loads(line) for line in TYPESAFE_SELECTION.read_text().splitlines() if line.strip()]
+    payloads: dict[str, dict] = {}
+    for workflow in sorted({item["upstream"]["workflow"] for item in selection}):
+        text = _fetch(TYPESAFE_CASES.format(workflow=workflow)).decode()
+        prefix = "__VIEWER_DATA__("
+        if not text.startswith(prefix):
+            raise ValueError(f"unexpected wrapper in TypeSafe {workflow} payload")
+        payload, _ = json.JSONDecoder().raw_decode(text[len(prefix):])
+        digest = hashlib.sha256(json.dumps(payload, indent=2).encode()).hexdigest()
+        pinned = {item["upstream"]["snapshot_sha256"] for item in selection if item["upstream"]["workflow"] == workflow}
+        if digest not in pinned:
+            raise ValueError(f"TypeSafe {workflow} payload changed since the selection was frozen")
+        payloads[workflow] = payload["eval"]
+
+    records = []
+    for item in selection:
+        up = item["upstream"]
+        evaluation = payloads[up["workflow"]]
+        case = evaluation["cases"][up["case_id"]]
+        source_question = evaluation["questions"][up["question_index"]]
+        if source_question["type"] == "noul":
+            criteria = source_question.get("criteria") or {}
+            question = {"type": "noul", "instructions": source_question["instructions"],
+                        "criteria": {"true": criteria.get("true"), "false": criteria.get("false")}}
+            keys, source_keys = ["yes", "no"], ["true", "false"]
+        elif source_question["type"] == "choice":
+            question = {"type": "choice", "instructions": source_question["instructions"],
+                        "criteria": source_question["criteria"]}
+            keys = source_keys = list(question["criteria"])
+        else:
+            raise ValueError("the frozen selection excludes score questions")
+        if source_keys != item["option_ids"]:
+            raise ValueError(f"option order changed for {item['id']}")
+
+        sets = case["reference_answers"][up["node"]][up["question_id"]]["sets"]
+        reference = dict(zip(keys, _mean_reference(sets, source_keys)))
+        top = max(reference, key=reference.get)
+        if sum(1 for v in reference.values() if abs(v - reference[top]) < 1e-12) != 1:
+            raise ValueError(f"reference argmax is not unique for {item['id']}")
+        published = next(
+            (node["answers"][up["question_id"]] for node in case["models"]["typesafe"]["nodes"]
+             if node["node"] == up["node"] and node.get("doc") == up["document_index"]
+             and node.get("questions", {}).get(up["question_id"]) == up["question_index"]
+             and up["question_id"] in node.get("answers", {})),
+            None,
+        )
+        if published is None:
+            raise ValueError(f"no published Jev answer for {item['id']}")
+        records.append({
+            "family": item["family"],
+            "source_id": item["id"],
+            "state": evaluation["documents"][up["document_index"]],
+            "questions": {up["question_id"]: question},
+            "labels": {up["question_id"]: (top == "yes") if question["type"] == "noul" else top},
+            "references": {up["question_id"]: {"jev": answer_distribution(published, keys), "reference": reference}},
+        })
+    if len(records) != 102:
+        raise ValueError(f"expected 102 TypeSafe rows, built {len(records)}")
+    return records
+
+
+def _mean_reference(sets: list[dict], keys: list[str]) -> list[float]:
+    total = [0.0] * len(keys)
+    for answer in sets:
+        if answer.get("probabilities"):
+            raw = {str(k): float(v) for k, v in answer["probabilities"].items()}
+            norm = sum(raw.get(k, 0.0) for k in keys)
+            for i, key in enumerate(keys):
+                total[i] += raw.get(key, 0.0) / norm
+        else:
+            value = answer["value"]
+            key = str(value).lower() if isinstance(value, bool) else str(value)
+            total[keys.index(key)] += 1.0
+    return [value / len(sets) for value in total]
+
+
+def build_every() -> list[dict]:
+    """Every's parallel judgment lab: Jev's recorded answers on identical inputs.
+
+    One record per document with all of that experiment's questions. Gold
+    labels exist for the judge grid and the two retrieval grids (author
+    labels); the other experiments carry only Jev's answers, so they measure
+    agreement. The AI-writing checker is skipped because only excerpts of its
+    documents are published.
+    """
+    data = json.loads(_fetch(EVERY_EXPERIMENTS))
+    if data["generated_at"] != EVERY_EXPERIMENTS_GENERATED_AT:
+        raise ValueError(f"Every experiments file changed: generated_at is {data['generated_at']}")
+    archive_bytes = _fetch(EVERY_ARCHIVE)
+    if hashlib.sha256(archive_bytes).hexdigest() != EVERY_ARCHIVE_SHA256:
+        raise ValueError("Every source archive changed")
+    archive = zipfile.ZipFile(io.BytesIO(archive_bytes))
+    script = archive.read("scripts/run-experiments.mjs").decode()
+    policy = json.loads(re.search(r'const SUPPORT_POLICY = ("[^\n]+");', script).group(1))
+    experiments = {e["id"]: e for e in data["experiments"]}
+    records: list[dict] = []
+
+    grid = experiments["judge-grid"]
+    cells = {(c["output"], c["question"]): c for c in grid["typesafe"]["cells"]}
+    for document in grid["documents"]:
+        questions = {qid: {"type": "noul", "instructions": text} for qid, text in grid["questions"].items()}
+        records.append({
+            "family": "every_judge-grid",
+            "state": {"policy": policy, "candidate_response": document["response"]},
+            "questions": questions,
+            "labels": {qid: bool(document["expected"][qid]) for qid in questions},
+            "references": {qid: {"jev": _noul(cells[document["id"], qid]["value"])} for qid in questions},
+        })
+
+    for name, folder in (("code-rag", "code-repo"), ("company-brain", "company-brain")):
+        by_document: dict[str, dict] = {}
+        for query in experiments[name]["grid"]:
+            for cell in query["cells"]:
+                entry = by_document.setdefault(cell["document"], {"questions": {}, "labels": {}, "references": {}})
+                entry["questions"][query["id"]] = {"type": "noul", "instructions": query["question"]}
+                entry["labels"][query["id"]] = bool(cell["relevant"])
+                entry["references"][query["id"]] = {"jev": _noul(cell["value"])}
+        for document, entry in by_document.items():
+            records.append({"family": f"every_{name}", "state": archive.read(f"fixtures/{folder}/{document}").decode(), **entry})
+
+    for name in ("action-firewall", "vc-diligence"):
+        experiment = experiments[name]
+        questions = {qid: {"type": "noul", "instructions": text} for qid, text in experiment["questions"].items()}
+        for row in experiment["rows"]:
+            records.append({"family": f"every_{name}", "state": row["text"], "questions": questions,
+                            "references": {qid: {"jev": _noul(row["signals"][qid])} for qid in questions}})
+
+    for name in ("customer-voice", "ceo-radar", "inbox-urgency", "persona-panel"):
+        experiment = experiments[name]
+        questions = {qid: ({"type": "noul", "instructions": spec} if isinstance(spec, str) else spec)
+                     for qid, spec in experiment["questions"].items()}
+        for row in experiment["rows"]:
+            references = {qid: {"jev": answer_distribution(answer, _keys(questions[qid]))}
+                          for qid, answer in row["typed_answers"].items()}
+            records.append({"family": f"every_{name}", "state": row["document"], "questions": questions,
+                            "references": references})
+    return records
+
+
+def _noul(probability: float) -> Distribution:
+    return {"yes": float(probability), "no": 1.0 - float(probability)}
+
+
+def _keys(question: dict) -> list[str]:
+    if question["type"] == "choice":
+        return list(question["criteria"])
+    if question["type"] == "score":
+        return [str(i) for i in range(len(question["criteria"]))]
+    return ["yes", "no"]
+
+
+def write(records: list[dict], output: Path) -> None:
+    output.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records))

ruling/calibration.py

@@ -0,0 +1,182 @@
+"""Temperature scaling, confidence, and calibration metrics.
+
+Restricted log-probabilities (one per allowed answer) are divided by a
+per-question-type temperature before the softmax. A temperature above 1
+flattens an overconfident model; below 1 sharpens an underconfident one.
+Temperatures are fit on labeled data by minimizing negative log-likelihood and
+stored with the model, revision, and scoring settings they were fit for, so a
+file cannot silently apply to a different model.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+
+QUESTION_TYPES = ("choice", "score", "noul")
+
+
+@dataclass(frozen=True)
+class Provenance:
+    model: str
+    revision: str | None
+    rotations: int
+    prior_debias: bool
+    adapter: str | None = None
+
+    def mismatch(self, other: "Provenance") -> list[str]:
+        return [name for name in ("model", "revision", "rotations", "prior_debias", "adapter")
+                if getattr(self, name) != getattr(other, name)]
+
+
+@dataclass
+class Calibration:
+    temperatures: dict[str, float] = field(default_factory=dict)
+    provenance: Provenance | None = None
+
+    def temperature(self, question_type: str) -> float:
+        return self.temperatures.get(question_type, 1.0)
+
+    @classmethod
+    def load(cls, path: Path) -> "Calibration":
+        data = json.loads(path.read_text())
+        temperatures = {k: float(v) for k, v in data["temperatures"].items()}
+        for question_type, value in temperatures.items():
+            if question_type not in QUESTION_TYPES or value <= 0:
+                raise ValueError(f"invalid temperature for {question_type!r}: {value}")
+        return cls(temperatures=temperatures, provenance=Provenance(**data["provenance"]))
+
+    def save(self, path: Path) -> None:
+        if self.provenance is None:
+            raise ValueError("a calibration cannot be saved without provenance")
+        data = {"temperatures": self.temperatures, "provenance": self.provenance.__dict__}
+        path.write_text(json.dumps(data, indent=2) + "\n")
+
+    def check(self, provenance: Provenance) -> None:
+        """Raise unless this calibration was fit for exactly this model and scoring setup."""
+        if self.provenance is None:
+            return
+        fields = self.provenance.mismatch(provenance)
+        if fields:
+            details = ", ".join(f"{f}: calibrated for {getattr(self.provenance, f)!r}, running {getattr(provenance, f)!r}" for f in fields)
+            raise ValueError(f"calibration does not match the running engine ({details})")
+
+
+def softmax(logits: np.ndarray, temperature: float = 1.0) -> np.ndarray:
+    scaled = np.asarray(logits, dtype=np.float64) / temperature
+    scaled -= scaled.max()
+    exp = np.exp(scaled)
+    return exp / exp.sum()
+
+
+def log_softmax(logits: np.ndarray) -> np.ndarray:
+    shifted = np.asarray(logits, dtype=np.float64) - np.max(logits)
+    return shifted - np.log(np.exp(shifted).sum())
+
+
+def choice_confidence(probabilities: np.ndarray) -> float:
+    """How far the leading option stands above a uniform distribution, from 0 to 1.
+
+    This and `score_confidence` reimplement the formulas TypeSafe publishes in
+    its MIT-licensed system-one-adapter, so a confidence value here means the
+    same thing as one from the hosted API.
+    """
+    probabilities = _normalized(probabilities)
+    if probabilities.size == 1:
+        return 1.0
+    uniform = 1.0 / probabilities.size
+    return float((probabilities.max() - uniform) / (1.0 - uniform))
+
+
+def score_confidence(probabilities: np.ndarray) -> float:
+    """How tightly a score's mass sits around its modal level, from 0 to 1."""
+    probabilities = _normalized(probabilities)
+    if probabilities.size == 1:
+        return 1.0
+    levels = np.arange(probabilities.size)
+    mode = int(probabilities.argmax())
+    spread = float((probabilities * np.abs(levels - mode)).sum())
+    uniform_spread = float(np.abs(levels - (probabilities.size - 1) / 2).mean())
+    return max(0.0, 1.0 - spread / uniform_spread)
+
+
+def _normalized(probabilities: np.ndarray) -> np.ndarray:
+    probabilities = np.asarray(probabilities, dtype=np.float64)
+    total = probabilities.sum()
+    return np.full(probabilities.size, 1.0 / probabilities.size) if total == 0 else probabilities / total
+
+
+def negative_log_likelihood(logits: list[np.ndarray], labels: list[int], temperature: float) -> float:
+    total = 0.0
+    for row, label in zip(logits, labels):
+        total -= math.log(max(softmax(row, temperature)[label], 1e-12))
+    return total / len(labels)
+
+
+def fit_temperature(logits: list[np.ndarray], labels: list[int]) -> float:
+    """Golden-section search over log-temperature for the NLL minimum."""
+    if not logits:
+        raise ValueError("cannot fit a temperature without examples")
+    lo, hi = math.log(0.05), math.log(20.0)
+    ratio = (math.sqrt(5) - 1) / 2
+    a = hi - ratio * (hi - lo)
+    b = lo + ratio * (hi - lo)
+    fa = negative_log_likelihood(logits, labels, math.exp(a))
+    fb = negative_log_likelihood(logits, labels, math.exp(b))
+    for _ in range(60):
+        if fa < fb:
+            hi, b, fb = b, a, fa
+            a = hi - ratio * (hi - lo)
+            fa = negative_log_likelihood(logits, labels, math.exp(a))
+        else:
+            lo, a, fa = a, b, fb
+            b = lo + ratio * (hi - lo)
+            fb = negative_log_likelihood(logits, labels, math.exp(b))
+    return math.exp((lo + hi) / 2)
+
+
+def coverage_at_error(confidence: list[float], correct: list[bool], budget: float) -> float:
+    """The largest share of decisions acceptable in descending-confidence order while the
+    error rate among the accepted stays within `budget`.
+
+    This is jev-benchmarks' selective-automation metric. Equal confidences are admitted as
+    a whole group, so a permutation of tied rows cannot change the answer. A model that
+    is confidently wrong scores low here whatever its accuracy; a temperature cannot move
+    it, because the order is what is measured.
+    """
+    if not 0 <= budget <= 1:
+        raise ValueError("error budget must be within [0, 1]")
+    confidence, correct = np.asarray(confidence, dtype=np.float64), np.asarray(correct, dtype=bool)
+    if confidence.size == 0:
+        return 0.0
+    order = np.argsort(-confidence, kind="stable")
+    ranked, errors = confidence[order], np.cumsum(~correct[order])
+    group_ends = np.r_[np.flatnonzero(ranked[1:] != ranked[:-1]), ranked.size - 1]
+    accepted = group_ends + 1
+    within = np.flatnonzero(errors[group_ends] <= budget * accepted)
+    return float(accepted[within[-1]] / ranked.size) if within.size else 0.0
+
+
+def confident_error_rate(confidence: list[float], correct: list[bool], threshold: float = 0.9) -> float:
+    """The share of all decisions that were wrong while reporting at least `threshold`."""
+    confidence, correct = np.asarray(confidence, dtype=np.float64), np.asarray(correct, dtype=bool)
+    return float(np.mean((confidence >= threshold) & ~correct)) if confidence.size else 0.0
+
+
+def expected_calibration_error(top_probabilities: list[float], correct: list[bool], bins: int = 10) -> float:
+    """Gap between stated top-1 probability and observed accuracy, weighted by bin size."""
+    if not top_probabilities:
+        return 0.0
+    probs = np.asarray(top_probabilities, dtype=np.float64)
+    hits = np.asarray(correct, dtype=np.float64)
+    edges = np.linspace(0.0, 1.0, bins + 1)
+    error = 0.0
+    for lo, hi in zip(edges[:-1], edges[1:]):
+        mask = (probs > lo) & (probs <= hi) if lo > 0 else (probs >= lo) & (probs <= hi)
+        if mask.any():
+            error += mask.mean() * abs(probs[mask].mean() - hits[mask].mean())
+    return float(error)

ruling/cascade.py

@@ -0,0 +1,83 @@
+"""Two engines behind one answer: a small one for every question, a larger one for the doubtful ones.
+
+The first engine scores the whole request. Any question whose calibrated top
+probability falls under the threshold is scored again by the second engine, and
+that answer replaces the first. Measured on TypeSafe's and Every's public rows
+with a threshold fitted on one set and applied to the other, the second engine
+ran on about 13% of questions and the pair matched its accuracy alone.
+
+Each engine's own temperature is applied here, so the logits handed back are
+already calibrated and the cascade carries an identity calibration of its own.
+
+When the second engine is a host that is overloaded or unreachable, the first
+engine's answers stand for the questions that would have been escalated: the
+request degrades to the primary's own confidence instead of failing.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+
+from ruling.calibration import Calibration, Provenance, softmax
+from ruling.engine import RawScore, Scored, answer
+from ruling.hosted import HostUnavailable
+from ruling.questions import Choice, Noul, Score, State, SystemOneRequest, SystemOneResponse, Usage
+
+PREFIX = "cascade:"
+log = logging.getLogger("ruling")
+
+
+class CascadeEngine:
+    """Same interface as the engines it wraps, so the server and the harness need no special case."""
+
+    prior_debias = False
+
+    def __init__(self, first, second, threshold: float):
+        if not 0 <= threshold <= 1:
+            raise ValueError("cascade threshold must be a probability")
+        self.first, self.second, self.threshold = first, second, threshold
+        self.model_id = f"{PREFIX}{first.model_id}>{second.model_id}"
+        self.revision = None
+        self.adapter = first.adapter
+        self.rotations = first.rotations
+        self.max_input_tokens = min(first.max_input_tokens, second.max_input_tokens)
+        self.max_branch_tokens = min(first.max_branch_tokens, second.max_branch_tokens)
+        self.max_options = min(first.max_options, second.max_options)
+        self.reads_images = first.reads_images and second.reads_images
+        self.calibration = Calibration()
+
+    @property
+    def provenance(self) -> Provenance:
+        return Provenance(self.model_id, self.revision, self.rotations, self.prior_debias, self.adapter)
+
+    def score(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        first = self.first.score(state, questions)
+        raw = {qid: self._tempered(self.first, q, first.raw[qid]) for qid, q in questions.items()}
+        doubtful = {qid: q for qid, q in questions.items() if softmax(raw[qid].logits).max() < self.threshold}
+        tokens = first.input_tokens
+        if doubtful:
+            try:
+                second = self.second.score(state, doubtful)
+            except HostUnavailable as exc:
+                log.warning("cascade kept %s's answers for %d doubtful questions: %s", self.first.model_id, len(doubtful), exc)
+                return Scored(raw=raw, input_tokens=tokens)
+            tokens += second.input_tokens
+            for qid, q in doubtful.items():
+                raw[qid] = self._tempered(self.second, q, second.raw[qid])
+        return Scored(raw=raw, input_tokens=tokens)
+
+    @staticmethod
+    def _tempered(engine, question: Choice | Score | Noul, raw: RawScore) -> RawScore:
+        """The engine's logits with its own fitted temperature already divided out."""
+        temperature = engine.calibration.temperature(question.type)
+        return RawScore(keys=raw.keys, logits=np.asarray(raw.logits, dtype=np.float64) / temperature)
+
+    def evaluate(self, request: SystemOneRequest) -> SystemOneResponse:
+        scored = self.score(request.state, request.questions)
+        answers = {qid: answer(request.questions[qid], scored.raw[qid], self.calibration) for qid in request.questions}
+        return SystemOneResponse(model=self.model_id, answers=answers, usage=Usage(input_tokens=scored.input_tokens))
+
+    def answer(self, question, raw: RawScore):
+        return answer(question, raw, self.calibration)