ruling 0.2.0__py3-none-any.whl

ruling/modernbert.py

@@ -0,0 +1,163 @@
+"""ModernBERT in MLX: a bidirectional encoder, not a generative model.
+
+A port of Hugging Face's ModernBertForMaskedLM that takes explicit position
+ids and a full boolean attention mask, so a caller can pack several
+independent segments into one sequence. Every third layer attends globally;
+the others see tokens within half of `local_attention` positions. Weights load
+from the checkpoint's safetensors; the input embedding is tied to the decoder.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import mlx.core as mx
+import mlx.nn as nn
+from huggingface_hub import hf_hub_download
+from transformers import AutoTokenizer
+
+
+@dataclass(frozen=True)
+class Config:
+    vocab_size: int
+    hidden_size: int
+    num_hidden_layers: int
+    num_attention_heads: int
+    intermediate_size: int
+    global_attn_every_n_layers: int
+    local_attention: int
+    global_rope_theta: float
+    local_rope_theta: float
+    norm_eps: float
+    max_position_embeddings: int
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Config":
+        return cls(**{name: data[name] for name in cls.__dataclass_fields__})
+
+    def is_global(self, layer: int) -> bool:
+        return layer % self.global_attn_every_n_layers == 0
+
+
+def _rotate_half(x: mx.array) -> mx.array:
+    half = x.shape[-1] // 2
+    return mx.concatenate([-x[..., half:], x[..., :half]], axis=-1)
+
+
+class Attention(nn.Module):
+    def __init__(self, config: Config, theta: float):
+        super().__init__()
+        self.heads = config.num_attention_heads
+        self.head_dim = config.hidden_size // config.num_attention_heads
+        self.Wqkv = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=False)
+        self.Wo = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
+        self._inv_freq = 1.0 / (theta ** (mx.arange(0, self.head_dim, 2, dtype=mx.float32) / self.head_dim))
+
+    def __call__(self, x: mx.array, positions: mx.array, mask: mx.array) -> mx.array:
+        batch, length, _ = x.shape
+        qkv = self.Wqkv(x).reshape(batch, length, 3, self.heads, self.head_dim)
+        q, k, v = (qkv[:, :, i].transpose(0, 2, 1, 3) for i in range(3))
+        freqs = positions[:, :, None].astype(mx.float32) * self._inv_freq
+        emb = mx.concatenate([freqs, freqs], axis=-1)[:, None]
+        cos, sin = mx.cos(emb).astype(q.dtype), mx.sin(emb).astype(q.dtype)
+        q = q * cos + _rotate_half(q) * sin
+        k = k * cos + _rotate_half(k) * sin
+        out = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.head_dim ** -0.5, mask=mask[:, None])
+        return self.Wo(out.transpose(0, 2, 1, 3).reshape(batch, length, -1))
+
+
+class MLP(nn.Module):
+    def __init__(self, config: Config):
+        super().__init__()
+        self.Wi = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False)
+        self.Wo = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+    def __call__(self, x: mx.array) -> mx.array:
+        value, gate = mx.split(self.Wi(x), 2, axis=-1)
+        return self.Wo(nn.gelu(value) * gate)
+
+
+class Layer(nn.Module):
+    def __init__(self, config: Config, index: int):
+        super().__init__()
+        self.is_global = config.is_global(index)
+        self.window = config.local_attention // 2
+        self.attn_norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps, bias=False) if index else None
+        self.attn = Attention(config, config.global_rope_theta if self.is_global else config.local_rope_theta)
+        self.mlp_norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps, bias=False)
+        self.mlp = MLP(config)
+
+    def __call__(self, x: mx.array, positions: mx.array, full: mx.array, local: mx.array) -> mx.array:
+        normed = self.attn_norm(x) if self.attn_norm is not None else x
+        x = x + self.attn(normed, positions, full if self.is_global else local)
+        return x + self.mlp(self.mlp_norm(x))
+
+
+class Encoder(nn.Module):
+    def __init__(self, config: Config):
+        super().__init__()
+        self.config = config
+        self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.embed_norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps, bias=False)
+        self.layers = [Layer(config, i) for i in range(config.num_hidden_layers)]
+        self.final_norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps, bias=False)
+
+    def __call__(self, tokens: mx.array, positions: mx.array, mask: mx.array) -> mx.array:
+        """Hidden states [batch, length, hidden]. `mask[b, i, j]` is True where token i may attend to token j."""
+        distance = mx.abs(positions[:, :, None] - positions[:, None, :])
+        local = mask & (distance <= self.config.local_attention // 2)
+        x = self.embed_norm(self.tok_embeddings(tokens))
+        for layer in self.layers:
+            x = layer(x, positions, mask, local)
+        return self.final_norm(x)
+
+
+class MaskedLM(nn.Module):
+    def __init__(self, config: Config):
+        super().__init__()
+        self.encoder = Encoder(config)
+        self.head_dense = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
+        self.head_norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps, bias=False)
+        self.decoder_bias = mx.zeros((config.vocab_size,))
+
+    def transform(self, hidden: mx.array) -> mx.array:
+        """The pretrained prediction transform applied before the vocabulary projection."""
+        return self.head_norm(nn.gelu(self.head_dense(hidden)))
+
+    def token_logits(self, hidden: mx.array) -> mx.array:
+        return self.encoder.tok_embeddings.as_linear(self.transform(hidden)) + self.decoder_bias
+
+
+RENAMES = {
+    "decoder.weight": "encoder.tok_embeddings.weight",
+    "decoder.bias": "decoder_bias",
+    "head.dense.weight": "head_dense.weight",
+    "head.norm.weight": "head_norm.weight",
+    "model.embeddings.norm.weight": "encoder.embed_norm.weight",
+    "model.final_norm.weight": "encoder.final_norm.weight",
+}
+
+
+def _rename(key: str) -> str:
+    return RENAMES.get(key) or key.replace("model.layers.", "encoder.layers.")
+
+
+def load(repo: str, revision: str) -> tuple[MaskedLM, AutoTokenizer, Config]:
+    config = Config.from_dict(json.loads(Path(hf_hub_download(repo, "config.json", revision=revision)).read_text()))
+    model = MaskedLM(config)
+    weights = mx.load(hf_hub_download(repo, "model.safetensors", revision=revision))
+    model.load_weights([(_rename(k), v) for k, v in weights.items()], strict=True)
+    mx.eval(model.parameters())
+    tokenizer = AutoTokenizer.from_pretrained(repo, revision=revision)
+    return model, tokenizer, config
+
+
+def sequence_inputs(token_lists: list[list[int]], pad_id: int) -> tuple[mx.array, mx.array, mx.array]:
+    """Plain batched inputs: positions from zero and a padding mask, as the reference model uses."""
+    width = max(len(t) for t in token_lists)
+    tokens = mx.array([t + [pad_id] * (width - len(t)) for t in token_lists])
+    positions = mx.broadcast_to(mx.arange(width)[None], (len(token_lists), width))
+    valid = mx.array([[i < len(t) for i in range(width)] for t in token_lists])
+    return tokens, positions, valid[:, None, :] & valid[:, :, None]

ruling/peft.py

@@ -0,0 +1,144 @@
+"""Convert a ruling adapter to and from PEFT's LoRA layout.
+
+mlx-lm stores `lora_a` as `(in, r)` and `lora_b` as `(r, out)` and applies
+`scale * (x @ lora_a) @ lora_b`. PEFT stores `lora_A.weight` as `(r, in)` and
+`lora_B.weight` as `(out, r)` and applies `(lora_alpha / r) * lora_B(lora_A(x))`.
+The conversion is therefore a transpose of each tensor and `lora_alpha = scale * r`.
+
+Only plain per-layer linears convert. mlx-lm can also put LoRA on switch
+(MoE) linears and embeddings, and multimodal wrappers nest the stack under
+`language_model`; none of those has a one-to-one PEFT key, so they are refused
+rather than exported under a name PEFT would silently ignore.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from pathlib import Path
+
+import mlx.core as mx
+from huggingface_hub import hf_hub_download
+
+MLX_KEY = re.compile(r"^model\.layers\.(\d+)\.(.+)\.lora_([ab])$")
+PEFT_KEY = re.compile(r"^base_model\.model\.model\.layers\.(\d+)\.(.+)\.lora_([AB])\.weight$")
+MLX_WEIGHTS = "adapters.safetensors"
+PEFT_WEIGHTS = "adapter_model.safetensors"
+CONFIG = "adapter_config.json"
+TRAINING = "training.json"
+
+
+def export_adapter(adapter: Path, output: Path, base_model: str) -> None:
+    """Write `adapter`, a directory from `ruling train`, as a PEFT adapter for `base_model`."""
+    config = json.loads((adapter / CONFIG).read_text())
+    if config.get("fine_tune_type", "lora") != "lora":
+        raise ValueError(f"only LoRA adapters export to PEFT, not {config['fine_tune_type']}")
+    lora = config["lora_parameters"]
+    rank = lora["rank"]
+    modules = _modules(mx.load(str(adapter / MLX_WEIGHTS)), MLX_KEY, "a", "b", rank, transposed=False)
+    layers, keys = _product(modules)
+    weights = {}
+    for (layer, key), (a, b) in modules.items():
+        prefix = f"base_model.model.model.layers.{layer}.{key}"
+        weights[f"{prefix}.lora_A.weight"] = a.T.astype(mx.float32)
+        weights[f"{prefix}.lora_B.weight"] = b.T.astype(mx.float32)
+    alpha = lora["scale"] * rank
+    peft_config = {
+        "peft_type": "LORA",
+        "task_type": "CAUSAL_LM",
+        "base_model_name_or_path": base_model,
+        "revision": None,
+        "inference_mode": False,
+        "r": rank,
+        "lora_alpha": int(alpha) if alpha == int(alpha) else alpha,
+        "lora_dropout": lora["dropout"],
+        "target_modules": sorted({key.rsplit(".", 1)[-1] for key in keys}),
+        "layers_to_transform": layers,
+        "bias": "none",
+    }
+    _write(output, PEFT_WEIGHTS, weights, peft_config, adapter / TRAINING)
+
+
+def import_adapter(peft: Path, output: Path) -> None:
+    """Write `peft`, a PEFT LoRA adapter directory, in the layout `ruling train` produces and the engine serves."""
+    config = json.loads((peft / CONFIG).read_text())
+    _check_peft_config(config)
+    rank = config["r"]
+    modules = _modules(mx.load(str(peft / PEFT_WEIGHTS)), PEFT_KEY, "A", "B", rank, transposed=True)
+    layers, keys = _product(modules)
+    total = base_layer_count(config["base_model_name_or_path"], config.get("revision"))
+    if layers != list(range(total - len(layers), total)):
+        raise ValueError(f"LoRA layers {layers} are not the last {len(layers)} of {total}; "
+                         "mlx-lm can only adapt a suffix of the stack")
+    weights = {}
+    for (layer, key), (a, b) in modules.items():
+        weights[f"model.layers.{layer}.{key}.lora_a"] = a.T
+        weights[f"model.layers.{layer}.{key}.lora_b"] = b.T
+    mlx_config = {
+        "fine_tune_type": "lora",
+        "num_layers": len(layers),
+        "lora_parameters": {"rank": rank, "scale": config["lora_alpha"] / rank, "dropout": config.get("lora_dropout", 0.0),
+                            "keys": keys},
+    }
+    _write(output, MLX_WEIGHTS, weights, mlx_config, peft / TRAINING)
+
+
+def base_layer_count(base_model: str, revision: str | None) -> int:
+    """How many decoder layers the base model has, from its Hugging Face `config.json`."""
+    config = json.loads(Path(hf_hub_download(base_model, "config.json", revision=revision)).read_text())
+    if "num_hidden_layers" not in config:
+        raise ValueError(f"{base_model}'s config.json has no top-level num_hidden_layers, so the LoRA layer "
+                         "range cannot be checked against the stack")
+    return config["num_hidden_layers"]
+
+
+def _modules(weights: dict[str, mx.array], pattern: re.Pattern, first: str, second: str, rank: int,
+             transposed: bool) -> dict[tuple[int, str], tuple[mx.array, mx.array]]:
+    """Pair each module's two LoRA factors by (layer, module path), refusing any key outside the pattern."""
+    halves: dict[tuple[int, str], dict[str, mx.array]] = {}
+    for name, tensor in weights.items():
+        match = pattern.match(name)
+        if match is None:
+            raise ValueError(f"{name!r} is not a LoRA factor on a plain per-layer linear; only "
+                             "model.layers.N.<module> adapters convert")
+        layer, key, factor = int(match.group(1)), match.group(2), match.group(3)
+        halves.setdefault((layer, key), {})[factor] = tensor
+    modules = {}
+    for (layer, key), pair in sorted(halves.items()):
+        if set(pair) != {first, second}:
+            raise ValueError(f"layer {layer} {key} has only one of its two LoRA factors")
+        a, b = pair[first], pair[second]
+        rank_axis = (a.shape[0], b.shape[1]) if transposed else (a.shape[1], b.shape[0])
+        if rank_axis != (rank, rank):
+            raise ValueError(f"layer {layer} {key} has rank {rank_axis}, but the config says {rank}")
+        modules[(layer, key)] = (a, b)
+    return modules
+
+
+def _product(modules: dict[tuple[int, str], object]) -> tuple[list[int], list[str]]:
+    """The layer indices and module paths, which must form a full product for either layout to express them."""
+    layers = sorted({layer for layer, _ in modules})
+    keys = sorted({key for _, key in modules})
+    missing = [(layer, key) for layer in layers for key in keys if (layer, key) not in modules]
+    if missing:
+        raise ValueError(f"every adapted layer must adapt the same modules; missing {missing}")
+    return layers, keys
+
+
+def _check_peft_config(config: dict) -> None:
+    if config.get("peft_type") != "LORA":
+        raise ValueError(f"only LORA adapters import, not {config.get('peft_type')}")
+    for field, plain in (("use_dora", False), ("use_rslora", False), ("bias", "none"), ("rank_pattern", {}),
+                         ("alpha_pattern", {}), ("fan_in_fan_out", False)):
+        if config.get(field, plain) not in (plain, None):
+            raise ValueError(f"PEFT adapters with {field}={config[field]!r} have no mlx-lm equivalent")
+
+
+def _write(output: Path, weights_name: str, weights: dict[str, mx.array], config: dict, training: Path) -> None:
+    output.mkdir(parents=True, exist_ok=True)
+    mx.save_safetensors(str(output / weights_name), weights)
+    (output / CONFIG).write_text(json.dumps(config, indent=2) + "\n")
+    # Provenance travels with the weights in both directions.
+    if training.exists():
+        shutil.copyfile(training, output / TRAINING)

ruling/playground.html

@@ -0,0 +1,217 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>ruling playground</title>
+<style>
+  :root { --bg: #fafaf7; --fg: #1b1b1b; --muted: #6b6b6b; --line: #dcdcd5; --bar: #2f6f4f; --accent: #b3451f; }
+  @media (prefers-color-scheme: dark) { :root { --bg: #161615; --fg: #ecece6; --muted: #9a9a92; --line: #33332f; --bar: #5fae85; --accent: #e0774d; } }
+  body { margin: 0; background: var(--bg); color: var(--fg); font: 15px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; }
+  main { max-width: 960px; margin: 0 auto; padding: 24px 16px; display: grid; gap: 16px; grid-template-columns: 1fr 1fr; }
+  h1 { grid-column: 1 / -1; margin: 0; font-size: 18px; }
+  h1 small { color: var(--muted); font-weight: normal; margin-left: 8px; }
+  label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 4px; }
+  textarea { width: 100%; box-sizing: border-box; min-height: 260px; background: transparent; color: inherit; border: 1px solid var(--line); border-radius: 4px; padding: 8px; font: inherit; }
+  button { grid-column: 1 / -1; justify-self: start; padding: 8px 14px; border: 1px solid var(--fg); background: var(--fg); color: var(--bg); border-radius: 4px; font: inherit; cursor: pointer; }
+  #out { grid-column: 1 / -1; display: grid; gap: 12px; }
+  .q { border: 1px solid var(--line); border-radius: 4px; padding: 10px 12px; }
+  .q b { display: block; margin-bottom: 6px; }
+  .row { display: grid; grid-template-columns: 140px 1fr 60px; gap: 8px; align-items: center; font-size: 13px; }
+  .bar { height: 10px; background: var(--line); border-radius: 2px; overflow: hidden; }
+  .bar i { display: block; height: 100%; background: var(--bar); }
+  .meta { color: var(--muted); font-size: 12px; }
+  .err { color: var(--accent); }
+  a { color: inherit; }
+  #thumbs { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 8px; }
+  .thumb { position: relative; width: 220px; max-width: 100%; font-size: 12px; color: var(--muted); }
+  /* The whole image, scaled to fit, so a receipt's text or a photo's edge stays readable before deciding. */
+  .thumb img { display: block; width: 100%; height: 220px; object-fit: contain; background: var(--line); border: 1px solid var(--line); border-radius: 4px; }
+  .thumb button { position: absolute; top: 2px; right: 2px; padding: 0 5px; font-size: 12px; line-height: 16px; }
+  input[type=file] { font: inherit; font-size: 12px; margin-top: 8px; max-width: 100%; }
+  #key, select { width: 100%; box-sizing: border-box; margin-bottom: 8px; background: transparent; color: inherit; border: 1px solid var(--line); border-radius: 4px; padding: 6px 8px; font: inherit; }
+  button.small { grid-column: auto; padding: 2px 8px; font-size: 12px; background: transparent; color: inherit; border-color: var(--line); }
+  @media (max-width: 700px) { main { grid-template-columns: 1fr; } }
+</style>
+</head>
+<body>
+<main>
+  <h1>ruling <small>typed decisions, no text generation · <a href="/history">retraining history</a></small></h1>
+  <div>
+    <label for="state">state (text or JSON)</label>
+    <textarea id="state">{"message": "My card was charged twice for one order. Refund the duplicate."}</textarea>
+    <label for="images">images, added to the state as <code>images</code>; ask about them as &lt;image 1&gt;, &lt;image 2&gt;, …</label>
+    <input type="file" id="images" accept="image/png,image/jpeg,image/webp" multiple>
+    <button type="button" id="image-example" class="small">image example</button>
+    <div id="images-note" class="meta"></div>
+    <div id="thumbs"></div>
+  </div>
+  <div>
+    <label for="questions">questions (JSON)</label>
+    <textarea id="questions">{
+  "team": {"type": "choice", "instructions": "Which team should handle this?",
+           "criteria": {"billing": "Charges and refunds", "technical": "Bugs", "sales": "Pricing"}},
+  "urgency": {"type": "score", "instructions": "How urgent is this?",
+              "criteria": ["Can wait", "This week", "Today"]},
+  "refund": {"type": "noul", "instructions": "The customer asks for a refund."}
+}</textarea>
+  </div>
+  <div>
+    <label for="model">model</label>
+    <select id="model"></select>
+    <label for="key">API key, only for a server started with RULING_API_KEY</label>
+    <input id="key" type="password" autocomplete="off">
+  </div>
+  <button id="run">Decide</button>
+  <div id="out"></div>
+</main>
+<script>
+const out = document.getElementById("out");
+const model = document.getElementById("model");
+const picker = document.getElementById("images");
+const thumbs = document.getElementById("thumbs");
+const stateBox = document.getElementById("state");
+const modalities = {};
+let picked = [];
+const keyBox = document.getElementById("key");
+// Kept for this tab only, and shared with the history page, which reads it from the same place.
+try { keyBox.value = sessionStorage.getItem("ruling-key") || ""; } catch {}
+function auth() {
+  const key = keyBox.value.trim();
+  return key ? { authorization: `Bearer ${key}` } : {};
+}
+keyBox.onchange = () => {
+  try { sessionStorage.setItem("ruling-key", keyBox.value.trim()); } catch {}
+  listModels();
+};
+function listModels() {
+  return fetch("/v1/models", { headers: auth() }).then(async r => {
+    const data = await r.json();
+    if (r.status === 401) throw new Error("this server needs its API key; enter it above");
+    if (!r.ok) throw new Error(`${r.status} ${data.error.code}: ${data.error.message}`);
+    return data;
+  }).then(showModels).catch(e => { out.innerHTML = `<div class="err">could not list models: ${e.message}</div>`; });
+}
+function showModels(data) {
+  const first = !model.options.length;
+  model.innerHTML = "";
+  out.innerHTML = "";
+  for (const m of data.models) {
+    modalities[m.name] = m.modalities;
+    const option = document.createElement("option");
+    option.value = m.name;
+    option.textContent = m.adapter ? `${m.name} (${m.base} + adapter ${m.adapter})` : m.name === m.base ? m.name : `${m.name} (${m.base})`;
+    model.appendChild(option);
+  }
+  showModalities();
+  // A server whose default model reads images opens on an example that uses one.
+  if (first && (modalities[model.value] || []).includes("image")) loadImageExample();
+}
+listModels();
+function parseState(text) { try { return JSON.parse(text); } catch { return text; } }
+function countImages(value) {
+  if (Array.isArray(value)) return value.reduce((n, v) => n + countImages(v), 0);
+  if (value && typeof value === "object") {
+    return value.type === "image" ? 1 : Object.values(value).reduce((n, v) => n + countImages(v), 0);
+  }
+  return 0;
+}
+// Picked images go after everything already in the state, so they take the next numbers.
+function withImages(state) {
+  if (!picked.length) return state;
+  const images = picked.map(({ media_type, data }) => ({ type: "image", media_type, data }));
+  return state && typeof state === "object" && !Array.isArray(state) ? { ...state, images } : { text: state, images };
+}
+function showThumbs() {
+  const first = countImages(parseState(stateBox.value)) + 1;
+  thumbs.innerHTML = "";
+  picked.forEach((image, i) => {
+    const box = document.createElement("div");
+    box.className = "thumb";
+    box.innerHTML = `<img alt=""><span>&lt;image ${first + i}&gt;</span><button type="button" aria-label="remove">×</button>`;
+    box.querySelector("img").src = `data:${image.media_type};base64,${image.data}`;
+    box.querySelector("button").onclick = () => { picked.splice(i, 1); showThumbs(); };
+    thumbs.appendChild(box);
+  });
+}
+function showModalities() {
+  const sees = (modalities[model.value] || []).includes("image");
+  picker.disabled = !sees;
+  document.getElementById("image-example").disabled = !sees;
+  document.getElementById("images-note").textContent = sees ? "" : "this model reads text only; serve a vision: model to ask about images";
+}
+// An example to try with a model that reads images: a receipt drawn here, so the page ships no image file.
+const IMAGE_STATE = {"note": "The customer says order 4471 was charged twice and attached the receipt."};
+const IMAGE_QUESTIONS = {
+  "kind": {"type": "choice", "instructions": "What is <image 1>?",
+           "criteria": {"receipt": "A store or restaurant receipt", "photo": "A photo of a person or a place",
+                        "screenshot": "A screenshot of an app"}},
+  "same_order": {"type": "noul", "instructions": "The order number on <image 1> is the one in the note."},
+  "total": {"type": "score", "instructions": "How large is the total on <image 1>?",
+            "criteria": ["Under $10", "$10 to $100", "Over $100"]}
+};
+function drawReceipt() {
+  const canvas = document.createElement("canvas");
+  canvas.width = 360; canvas.height = 480;
+  const g = canvas.getContext("2d");
+  g.fillStyle = "#fffdf6"; g.fillRect(0, 0, canvas.width, canvas.height);
+  g.fillStyle = "#222"; g.textBaseline = "top";
+  const lines = [["bold 26px sans-serif", "CORNER CAFE"], ["16px monospace", "12 Market St, Springfield"], ["16px monospace", ""],
+                 ["18px monospace", "Order #4471"], ["16px monospace", ""], ["18px monospace", "2 x Latte          9.00"],
+                 ["18px monospace", "1 x Bagel          4.50"], ["18px monospace", "1 x Fruit bowl     6.25"],
+                 ["18px monospace", "Tip                2.35"], ["16px monospace", ""], ["bold 22px monospace", "TOTAL  $22.10"],
+                 ["16px monospace", ""], ["18px monospace", "PAID  VISA ****1881"], ["16px monospace", "Thank you!"]];
+  let y = 24;
+  for (const [font, text] of lines) { g.font = font; g.fillText(text, 28, y); y += 30; }
+  return { media_type: "image/png", data: canvas.toDataURL("image/png").split(",", 2)[1] };
+}
+function loadImageExample() {
+  stateBox.value = JSON.stringify(IMAGE_STATE, null, 2);
+  document.getElementById("questions").value = JSON.stringify(IMAGE_QUESTIONS, null, 2);
+  picked = [drawReceipt()];
+  showThumbs();
+}
+document.getElementById("image-example").onclick = loadImageExample;
+model.onchange = showModalities;
+stateBox.oninput = showThumbs;
+picker.onchange = async () => {
+  const read = file => new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve({ media_type: file.type, data: reader.result.split(",", 2)[1] });
+    reader.onerror = () => reject(reader.error);
+    reader.readAsDataURL(file);
+  });
+  picked = picked.concat(await Promise.all([...picker.files].map(read)));
+  picker.value = "";
+  showThumbs();
+};
+function bar(label, p) {
+  return `<div class="row"><span>${label}</span><div class="bar"><i style="width:${(p * 100).toFixed(1)}%"></i></div><span>${p.toFixed(3)}</span></div>`;
+}
+document.getElementById("run").onclick = async () => {
+  out.innerHTML = '<div class="meta">deciding…</div>';
+  let questions;
+  try { questions = JSON.parse(document.getElementById("questions").value); }
+  catch (e) { out.innerHTML = `<div class="err">questions is not valid JSON: ${e.message}</div>`; return; }
+  const body = { model: model.value, state: withImages(parseState(stateBox.value)), questions };
+  const started = performance.now();
+  const res = await fetch("/v1/systemone", { method: "POST", headers: { "content-type": "application/json", ...auth() },
+                                             body: JSON.stringify(body) });
+  const ms = Math.round(performance.now() - started);
+  const data = await res.json();
+  if (!res.ok) { out.innerHTML = `<div class="err">${res.status} ${data.error.code}: ${data.error.message}</div>`; return; }
+  let html = `<div class="meta">${data.model} · ${data.usage.input_tokens} input tokens · ${ms} ms</div>`;
+  for (const [id, a] of Object.entries(data.answers)) {
+    html += `<div class="q"><b>${id} <span class="meta">${a.type}</span></b>`;
+    if (a.type === "noul") html += bar("yes", a.noul) + bar("no", 1 - a.noul);
+    else {
+      for (const [k, p] of Object.entries(a.probabilities)) html += bar(a.type === "score" ? a.legend[k] : k, p);
+      html += `<div class="meta">${a.type === "choice" ? "choice: " + a.choice : "score: " + a.score.toFixed(2)} · confidence ${a.confidence.toFixed(2)}</div>`;
+    }
+    html += "</div>";
+  }
+  out.innerHTML = html;
+};
+</script>
+</body>
+</html>

ruling/prompt.py

@@ -0,0 +1,134 @@
+"""Render state and questions into the text the model scores.
+
+The state is rendered once as a prefix; each question is rendered as a suffix.
+The engine encodes the prefix a single time and evaluates every suffix against
+it, which is what keeps per-question cost flat as questions are added.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+from ruling.questions import (
+    IMAGE_REFERENCE,
+    Choice,
+    Noul,
+    Score,
+    State,
+    question_statement,
+    render_text,
+    state_images,
+    without_images,
+)
+
+SYSTEM_PROMPT = (
+    "You are a decision engine embedded in software. You are shown a state and one "
+    "question about it. Answer with a single letter naming the best option. "
+    "Never write anything except that letter."
+)
+
+# Splits the rendered user message into the shared prefix and the per-question
+# suffix after the chat template has been applied. It must never occur in user data.
+SPLIT_MARKER = "\x00RULING_SPLIT\x00"
+
+# A content-free question used to measure how much the model prefers each
+# answer letter regardless of content (Zhao et al. 2021, contextual calibration).
+PRIOR_STATE = "No information is available."
+PRIOR_INSTRUCTIONS = "Pick an option."
+PRIOR_OPTION = "No description."
+
+
+@dataclass(frozen=True)
+class RenderedQuestion:
+    text: str
+    keys: list[str]
+    codes: list[str]
+    # The parts of `text`, for a layout that orders them differently.
+    question: str = ""
+    options: str = ""
+    closing: str = ""
+
+
+def render_state(state: State) -> str:
+    """The state as prompt text, each image as a reference such as `<image 1>`; an engine that reads
+    images supplies them beside the text."""
+    state = without_images(state)
+    body = state if isinstance(state, str) else json.dumps(state, indent=2, ensure_ascii=False)
+    return f"State:\n{body}\n\n"
+
+
+def render_question(question: Choice | Score | Noul, codes: list[str], rotation: int = 0) -> RenderedQuestion:
+    """Render one question under a given `rotation` of its answer order.
+
+    Choice and Noul rotate cyclically. Score has two orderings, lowest to
+    highest and highest to lowest, because level order is part of the question.
+    Averaging over orderings cancels the positional bias small models show
+    toward particular letters.
+    """
+    instructions = question_statement(question)
+    if isinstance(question, Choice):
+        keys = _rotate(list(question.criteria), rotation)
+        lines = [_option_line(code, key, question.criteria[key]) for code, key in zip(codes, keys)]
+        heading = "Options:"
+        closing = "Respond with only the letter of the best option."
+    elif isinstance(question, Score):
+        levels = list(enumerate(question.criteria))
+        if rotation % 2:
+            levels.reverse()
+            heading = "Levels, from highest to lowest:"
+        else:
+            heading = "Levels, from lowest to highest:"
+        keys = [str(i) for i, _ in levels]
+        lines = [_option_line(code, render_text(level), None) for code, (_, level) in zip(codes, levels)]
+        closing = "Respond with only the letter of the level that fits best."
+    else:
+        keys = _rotate(["yes", "no"], rotation)
+        descriptions = {"yes": None, "no": None}
+        if question.criteria is not None:
+            descriptions = {"yes": question.criteria.true, "no": question.criteria.false}
+        lines = [_option_line(code, key, descriptions[key]) for code, key in zip(codes, keys)]
+        heading = "Is the statement true?"
+        closing = "Respond with only the letter."
+    used = codes[: len(keys)]
+    question = f"Question: {instructions}"
+    options = "\n".join([heading, *lines])
+    text = "\n".join([question, options, closing])
+    return RenderedQuestion(text=text, keys=keys, codes=used, question=question, options=options, closing=closing)
+
+
+def hosted_user_message(state: State, rendered: RenderedQuestion) -> str | list[dict]:
+    """The user turn for an OpenAI-compatible host: the state, then the question,
+    its options and the answer cue, the layout the local engine scores.
+
+    The model answers differently when the question comes first: with the same
+    4-bit weights behind mlx_lm.server, that layout got 115 of authored144's
+    144 against 126 for this one. A host that caches prompt prefixes still
+    reuses the state across a request's questions and orderings, which is
+    the repeat a decision server sees.
+
+    A state with images becomes content parts, each image after the state text
+    and before the question, as the local vision engine lays them out.
+    """
+    images = state_images(state)
+    if not images:
+        return render_state(state) + rendered.text
+    # Each image follows the state under the reference the state text uses for it.
+    parts = [{"type": "text", "text": render_state(state)}]
+    for number, image in enumerate(images, 1):
+        parts += [{"type": "text", "text": f"{IMAGE_REFERENCE.format(number)}:"},
+                  {"type": "image_url", "image_url": {"url": image.data_url()}}]
+    return parts + [{"type": "text", "text": "\n\n" + rendered.text}]
+
+
+def prior_question(cardinality: int) -> Choice:
+    return Choice(instructions=PRIOR_INSTRUCTIONS, criteria={f"option{i + 1}": PRIOR_OPTION for i in range(cardinality)})
+
+
+def _rotate(keys: list[str], rotation: int) -> list[str]:
+    shift = rotation % len(keys)
+    return keys[shift:] + keys[:shift]
+
+
+def _option_line(code: str, label: str, description: object) -> str:
+    return f"{code}. {label}: {render_text(description)}" if description is not None else f"{code}. {label}"