ruling 0.2.0__tar.gz → 0.3.0__tar.gz

{ruling-0.2.0 → ruling-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ruling
-Version: 0.2.0
+Version: 0.3.0
 Summary: Open-weight recreation of TypeSafe AI's Jev: typed, probabilistic decisions from a local model, no text generation.
 Project-URL: Homepage, https://github.com/bradAGI/ruling
 Project-URL: Source, https://github.com/bradAGI/ruling
@@ -18,6 +18,8 @@ Requires-Dist: transformers>=5.17.0
 Requires-Dist: uvicorn>=0.30
 Provides-Extra: vision
 Requires-Dist: mlx-vlm>=0.7.8; extra == 'vision'
+Provides-Extra: wandb
+Requires-Dist: wandb>=0.19; extra == 'wandb'
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -52,15 +54,16 @@ is ahead. Not affiliated with TypeSafe AI.
 
 ## Installation
 
-You need an Apple Silicon Mac, Python 3.12+, and [uv](https://docs.astral.sh/uv/).
+You need an Apple Silicon Mac and Python 3.12+.
 
 ```bash
-git clone https://github.com/bradAGI/ruling && cd ruling
-uv sync                       # add --extra vision to ask about images
-uv run ruling serve           # downloads a 2.3 GB model on first start, then serves http://127.0.0.1:8010
+pip install ruling            # or: uv tool install ruling; add [vision] to ask about images
+ruling serve                  # downloads a 2.3 GB model on first start, then serves http://127.0.0.1:8010
 ```
 
-Open `http://127.0.0.1:8010/` for the playground. On NVIDIA, see
+Open `http://127.0.0.1:8010/` for the playground. To work on ruling itself,
+clone the repository and `uv sync`; see
+[Development](docs/development.md). On NVIDIA, see
 [vLLM](docs/reference.md#on-nvidia-vllm). Before relying on an answer, read the
 [limits](docs/getting-started.md#limits).
 

{ruling-0.2.0 → ruling-0.3.0}/README.md

@@ -30,15 +30,16 @@ is ahead. Not affiliated with TypeSafe AI.
 
 ## Installation
 
-You need an Apple Silicon Mac, Python 3.12+, and [uv](https://docs.astral.sh/uv/).
+You need an Apple Silicon Mac and Python 3.12+.
 
 ```bash
-git clone https://github.com/bradAGI/ruling && cd ruling
-uv sync                       # add --extra vision to ask about images
-uv run ruling serve           # downloads a 2.3 GB model on first start, then serves http://127.0.0.1:8010
+pip install ruling            # or: uv tool install ruling; add [vision] to ask about images
+ruling serve                  # downloads a 2.3 GB model on first start, then serves http://127.0.0.1:8010
 ```
 
-Open `http://127.0.0.1:8010/` for the playground. On NVIDIA, see
+Open `http://127.0.0.1:8010/` for the playground. To work on ruling itself,
+clone the repository and `uv sync`; see
+[Development](docs/development.md). On NVIDIA, see
 [vLLM](docs/reference.md#on-nvidia-vllm). Before relying on an answer, read the
 [limits](docs/getting-started.md#limits).
 

{ruling-0.2.0 → ruling-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ruling"
-version = "0.2.0"
+version = "0.3.0"
 description = "Open-weight recreation of TypeSafe AI's Jev: typed, probabilistic decisions from a local model, no text generation."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,6 +28,9 @@ ruling = "ruling.cli:main"
 vision = [
     "mlx-vlm>=0.7.8",
 ]
+wandb = [
+    "wandb>=0.19",
+]
 
 [dependency-groups]
 dev = [

{ruling-0.2.0 → ruling-0.3.0}/ruling/__init__.py

@@ -10,14 +10,17 @@ from ruling.questions import (
     SystemOneRequest,
     SystemOneResponse,
 )
-from ruling.sdk import RulingClient
+from ruling.sdk import Image, RetryPolicy, RulingClient, RulingError
 
 __all__ = [
     "Choice",
     "ChoiceAnswer",
+    "Image",
     "Noul",
     "NoulAnswer",
+    "RetryPolicy",
     "RulingClient",
+    "RulingError",
     "Score",
     "ScoreAnswer",
     "SystemOneRequest",

{ruling-0.2.0 → ruling-0.3.0}/ruling/cli.py

@@ -3,9 +3,12 @@
 from __future__ import annotations
 
 import argparse
+import contextlib
 import dataclasses
+import io
 import json
 import logging
+import sys
 from pathlib import Path
 
 import numpy as np
@@ -117,6 +120,9 @@ def main(argv: list[str] | None = None) -> None:
                        help="continue from the checkpoint in --output; only --steps may differ from the original run")
     train.add_argument("--from-adapter", type=Path,
                        help="start a fresh run from this adapter's weights; its LoRA settings must match")
+    train.add_argument("--wandb", nargs="?", const="", metavar="PROJECT",
+                       help="mirror every progress entry to a Weights & Biases run, in PROJECT or the account's default "
+                            "(needs `pip install 'ruling[wandb]'`)")
 
     export = commands.add_parser("export-adapter", help="write a ruling adapter in PEFT layout for transformers")
     export.add_argument("adapter", type=Path, help="a LoRA directory from `ruling train`")
@@ -162,6 +168,9 @@ def main(argv: list[str] | None = None) -> None:
 
     args = parser.parse_args(argv)
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
+    # Progress lines are read from log files while a run is going; block buffering would hold them back.
+    if isinstance(sys.stdout, io.TextIOWrapper):
+        sys.stdout.reconfigure(line_buffering=True)
     configure_memory(Settings.from_env())
 
     if args.command == "serve":
@@ -238,7 +247,7 @@ def main(argv: list[str] | None = None) -> None:
         print(f"wrote {len(records)} records to {args.output}")
         return
     if args.command == "train":
-        from ruling.train import Trainer, TrainConfig
+        from ruling.train import Trainer, TrainConfig, WandbSink
         options = {name.replace("-", "_"): getattr(args, name.replace("-", "_"))
                    for name in ("valid-fraction", "steps", "batch-size", "learning-rate", "max-grad-norm", "warmup", "lora-layers", "lora-rank",
                                 "lora-scale", "lora-dropout", "max-tokens", "eval-every", "valid-examples", "seed",
@@ -246,7 +255,11 @@ def main(argv: list[str] | None = None) -> None:
         config = TrainConfig(model=args.model or Settings.from_env().model, data=args.data, output=args.output,
                              holdout=args.holdout, grad_checkpoint=args.grad_checkpoint, from_adapter=args.from_adapter,
                              **{k: v for k, v in options.items() if v is not None})
-        Trainer(config, resume=args.resume).run()
+        # The sink is made before the model loads, so a missing package is reported at once.
+        sink = contextlib.nullcontext() if args.wandb is None else WandbSink(args.wandb or None, config)
+        trainer = Trainer(config, resume=args.resume)
+        with sink as entries:
+            trainer.run(sink=entries)
         return
     if args.command == "retrain":
         from ruling.ledger import DecisionLedger

{ruling-0.2.0 → ruling-0.3.0}/ruling/history.html

@@ -23,6 +23,8 @@
   .kept { color: var(--muted); }
   .serving { font-weight: bold; }
   .err { color: var(--accent); }
+  svg.curve { display: block; color: var(--bar); }
+  .kept svg.curve { color: var(--muted); }
 </style>
 </head>
 <body>
@@ -41,6 +43,23 @@ function text(value) {
   span.textContent = value;
   return span.innerHTML;
 }
+const WIDTH = 120, HEIGHT = 28, PAD = 2;
+function curve(training) {
+  // Validation loss over steps, both axes scaled to the run; a flat run or a lone point sits on the middle line.
+  const steps = training.map(e => e.step), losses = training.map(e => e.loss);
+  const x0 = Math.min(...steps), x1 = Math.max(...steps), y0 = Math.min(...losses), y1 = Math.max(...losses);
+  const x = s => x1 === x0 ? WIDTH / 2 : PAD + (s - x0) / (x1 - x0) * (WIDTH - 2 * PAD);
+  const y = l => y1 === y0 ? HEIGHT / 2 : HEIGHT - PAD - (l - y0) / (y1 - y0) * (HEIGHT - 2 * PAD);
+  const points = training.map(e => `${x(e.step).toFixed(1)},${y(e.loss).toFixed(1)}`).join(" ");
+  const last = training[training.length - 1];
+  return `<svg class="curve" viewBox="0 0 ${WIDTH} ${HEIGHT}" width="${WIDTH}" height="${HEIGHT}" role="img" ` +
+         `aria-label="validation loss ${pct(training[0].loss)} to ${pct(last.loss)} over ${last.step} steps">` +
+         `<polyline points="${points}" fill="none" stroke="currentColor" stroke-width="1.5"/></svg>`;
+}
+function finalLoss(training) {
+  const last = training[training.length - 1];
+  return `${pct(last.train_loss)} / ${pct(last.loss)}`;
+}
 let key = "";
 try { key = sessionStorage.getItem("ruling-key") || ""; } catch {}
 fetch("/v1/retrain", { headers: key ? { authorization: `Bearer ${key}` } : {} }).then(async r => {
@@ -59,13 +78,16 @@ fetch("/v1/retrain", { headers: key ? { authorization: `Bearer ${key}` } : {} })
     html += `<h2>${text(m.name)} <span class="meta">serving ${text(m.current ?? "the base model")}</span></h2>`;
     if (!m.versions.length) { html += '<div class="meta">no finished runs yet</div>'; continue; }
     html += '<div class="scroll"><table><thead><tr><th>version</th><th>from</th><th class="num">records</th>' +
-            '<th class="num">held-out accuracy</th><th class="num">held-out log loss</th><th>verdict</th></tr></thead><tbody>';
+            '<th class="num">held-out accuracy</th><th class="num">held-out log loss</th>' +
+            '<th>validation loss</th><th class="num">final loss train / valid</th><th>verdict</th></tr></thead><tbody>';
     for (const v of [...m.versions].reverse()) {
       const serving = v.version === m.current ? " serving" : "";
       html += `<tr class="${v.promoted ? "promoted" : "kept"}${serving}" data-version="${text(v.version)}">` +
               `<td>${text(v.version)}</td><td>${text(v.started_from ?? "base")}</td>` +
               `<td class="num">${v.records}</td><td class="num">${change(v.current, v.candidate, "accuracy")}</td>` +
               `<td class="num">${change(v.current, v.candidate, "nll")}</td>` +
+              `<td>${v.training?.length ? curve(v.training) : "–"}</td>` +
+              `<td class="num">${v.training?.length ? finalLoss(v.training) : "–"}</td>` +
               `<td>${v.promoted ? "promoted" : "kept"}: ${text(v.reason)}</td></tr>`;
     }
     html += "</tbody></table></div>";

{ruling-0.2.0 → ruling-0.3.0}/ruling/questions.py

@@ -27,6 +27,13 @@ MAX_QUESTIONS = 64
 MAX_IMAGES = 8
 MAX_IMAGE_BYTES = 5 * 2**20
 IMAGE_REFERENCE = "<image {}>"
+ImageMediaType = Literal["image/png", "image/jpeg", "image/webp"]
+IMAGE_SUFFIXES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp"}
+"""The media type an image file's suffix implies, for clients reading images off disk."""
+
+MAX_PREDICT_ROWS = 50_000
+"""Rows one `/v1/predict` or `/v1/evaluations` request may carry. A batch holds one admission slot
+for its whole duration, so the limit bounds how long one client can keep a slot."""
 
 # The hosted SDK sends its default model name on every request. Treating these as
 # "whichever model this server loaded" is what lets that client talk to ruling
@@ -110,7 +117,7 @@ def question_statement(question: Question) -> str:
 
 class Image(Strict):
     type: Literal["image"]
-    media_type: Literal["image/png", "image/jpeg", "image/webp"]
+    media_type: ImageMediaType
     data: str
 
     @field_validator("data")

{ruling-0.2.0 → ruling-0.3.0}/ruling/retrain.py

@@ -6,9 +6,13 @@ a relative link to the version being served (`RULING_ADAPTER=<dir>/current`).
 A run exports the ledger's resolved decisions, drops any whose state is in a
 held-out set, trains the next version (from `current`'s weights when there is
 one), scores both versions on the held-out sets, and repoints `current` only
-when the new one has lower log loss without losing accuracy. Every run leaves
-its records, its training log and the comparison in its version directory,
-promoted or not.
+when the new one has lower log loss without losing accuracy. Each version is
+scored at its own fitted temperature: training sharpens an adapter faster than
+it corrects it, and a version that is right more often must not lose for being
+overconfident. The candidate's temperatures are saved beside its weights, so
+`RULING_CALIBRATION=<dir>/current/calibration.json` serves the promoted
+version calibrated. Every run leaves its records, its training log and the
+comparison in its version directory, promoted or not.
 """
 
 from __future__ import annotations
@@ -21,16 +25,17 @@ from pathlib import Path
 
 import mlx.core as mx
 
-from ruling.calibration import Calibration
+from ruling.calibration import Calibration, Provenance
 from ruling.config import Settings
 from ruling.engine import ADAPTER_CONFIG
-from ruling.evals import Record, collect, decision_blocks, load_dataset
+from ruling.evals import Observation, Record, collect, decision_blocks, fit, load_dataset
 from ruling.ledger import DecisionLedger
 from ruling.registry import build_engine
-from ruling.train import Trainer, TrainConfig, _state_key
+from ruling.train import TRAINING, Trainer, TrainConfig, _state_key
 
 CURRENT = "current"
 REPORT = "retrain.json"
+CALIBRATION = "calibration.json"
 DECISIONS = "decisions.jsonl"
 VERSION = re.compile(r"v(\d+)")
 
@@ -45,6 +50,15 @@ def should_promote(current: dict, candidate: dict) -> tuple[bool, str]:
                   f"accuracy {current['accuracy']:.4f} -> {candidate['accuracy']:.4f}")
 
 
+def score(observations: list[Observation], provenance: Provenance) -> tuple[dict, Calibration]:
+    """Held-out metrics at the temperatures that fit these observations best, and that calibration.
+
+    A temperature never changes which answer wins, so accuracy is the raw model's; log loss
+    becomes a measure of how well the model ranks answers, not of how sharply it states them."""
+    calibration = fit(observations, provenance)
+    return decision_blocks(observations, calibration)["overall"], calibration
+
+
 def next_version(adapters: Path) -> Path:
     numbers = [int(m.group(1)) for p in adapters.iterdir() if (m := VERSION.fullmatch(p.name))] if adapters.exists() else []
     return adapters / f"v{max(numbers, default=0) + 1}"
@@ -90,8 +104,11 @@ def retrain(ledger: DecisionLedger, adapters: Path, model: str, holdout: list[Pa
     served = build_engine(replace(settings, model=model, adapter_path=start, calibration_path=None, cascade_to=None,
                                   failover_to=None))
     candidate = served.variant(version, Calibration())
-    scores = {name: decision_blocks(collect(engine, held_out), Calibration())["overall"]
+    scored = {name: score(collect(engine, held_out), engine.provenance)
               for name, engine in (("current", served), ("candidate", candidate))}
+    scored["candidate"][1].save(version / CALIBRATION)
+    scores = {name: {**metrics, "temperatures": calibration.temperatures}
+              for name, (metrics, calibration) in scored.items()}
     promoted, reason = should_promote(scores["current"], scores["candidate"])
     report = {"version": version.name, "started_from": start.name if start else None, "records": len(records),
               "dropped_held_out": dropped, "holdout": [str(p) for p in holdout], **scores,
@@ -104,17 +121,26 @@ def retrain(ledger: DecisionLedger, adapters: Path, model: str, holdout: list[Pa
 
 
 def history(adapters: Path) -> dict:
-    """Every finished run's report in version order, and the version `current` points at.
+    """Every finished run's report in version order, each with its training curve, and the version `current`
+    points at.
 
     A version still training, or one whose run failed, has no report and is left out."""
     link = adapters / CURRENT
-    reports = [json.loads((version / REPORT).read_text())
+    reports = [_report(version)
                for version in sorted((p for p in adapters.iterdir() if VERSION.fullmatch(p.name)),
                                      key=lambda p: int(p.name[1:]))
                if (version / REPORT).exists()] if adapters.exists() else []
     return {"current": link.resolve().name if link.exists() else None, "versions": reports}
 
 
+def _report(version: Path) -> dict:
+    """The version's report, with the history its trainer logged under `training` when the log is there."""
+    report = json.loads((version / REPORT).read_text())
+    if (version / TRAINING).exists():
+        report["training"] = json.loads((version / TRAINING).read_text())["history"]
+    return report
+
+
 def _lora_layout(adapter: Path | None) -> dict:
     """The LoRA settings to train with: the starting adapter's, which its weights only fit, else the trainer's defaults."""
     if adapter is None:

ruling-0.3.0/ruling/sdk.py

@@ -0,0 +1,177 @@
+"""Python client mirroring the shape of the hosted System One SDK, plus ruling's batch, evaluation and usage routes."""
+
+from __future__ import annotations
+
+import base64
+import gzip
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from time import sleep
+
+import httpx
+from pydantic import TypeAdapter
+
+from ruling.questions import (
+    IMAGE_SUFFIXES,
+    Answer,
+    Choice,
+    ModelInfo,
+    ModelList,
+    Noul,
+    Score,
+    State,
+    SystemOneRequest,
+    SystemOneResponse,
+)
+from ruling.questions import Image as ImageBlock
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8010"
+
+GZIP_THRESHOLD = 64 * 1024
+"""Bodies of at least this many bytes are sent gzipped. JSON rows compress several times over, and
+below this size the round trip is dominated by the model, not the bytes."""
+
+RETRIED_STATUSES = (429, 503)
+"""A full queue and an absent upstream are the server's two transient refusals."""
+
+ANSWER = TypeAdapter(Answer)
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """How a request that was turned away or never arrived is tried again: at most `max_retries`
+    more times, waiting the server's `Retry-After` when it gives one, else doubling from
+    `backoff_initial` seconds up to `backoff_max`."""
+
+    max_retries: int = 2
+    backoff_initial: float = 0.5
+    backoff_max: float = 5.0
+
+    def delay(self, attempt: int, retry_after: str | None) -> float:
+        if retry_after is not None:
+            return float(retry_after)
+        return min(self.backoff_initial * 2**attempt, self.backoff_max)
+
+
+class RulingError(Exception):
+    """A refusal from the server: its HTTP `status` and the `code`, `message` and `param` of its error body."""
+
+    def __init__(self, status: int, code: str, message: str, param: str | None = None):
+        super().__init__(f"{status} {code}: {message}" + (f" (param {param})" if param else ""))
+        self.status, self.code, self.message, self.param = status, code, message, param
+
+    @classmethod
+    def from_response(cls, response: httpx.Response) -> "RulingError":
+        try:
+            error = response.json()["error"]
+        except (ValueError, KeyError, TypeError):
+            # Something between the client and ruling answered, such as a proxy's HTML error page.
+            return cls(response.status_code, "http_error", response.text)
+        return cls(response.status_code, error["code"], error["message"], error.get("param"))
+
+
+class Image:
+    """Image blocks for a state, in the wire shape the server reads."""
+
+    @staticmethod
+    def from_bytes(data: bytes, media_type: str) -> dict:
+        block = ImageBlock(type="image", media_type=media_type, data=base64.b64encode(data).decode())
+        return block.model_dump()
+
+    @staticmethod
+    def from_path(path: str | os.PathLike) -> dict:
+        path = Path(path)
+        try:
+            media_type = IMAGE_SUFFIXES[path.suffix.lower()]
+        except KeyError:
+            raise ValueError(f"{path.name}: an image must be one of {', '.join(IMAGE_SUFFIXES)}, "
+                             f"got {path.suffix or 'no suffix'}") from None
+        return Image.from_bytes(path.read_bytes(), media_type)
+
+
+class RulingClient:
+    def __init__(self, base_url: str | None = None, api_key: str | None = None, timeout: float = 60.0,
+                 retry: RetryPolicy | None = None, gzip: bool = True,
+                 transport: httpx.BaseTransport | None = None):
+        self.base_url = base_url or os.environ.get("RULING_BASE_URL", DEFAULT_BASE_URL)
+        # The server's own RULING_API_KEY, so one environment configures both ends.
+        api_key = api_key or os.environ.get("RULING_API_KEY")
+        headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
+        self.retry = retry if retry is not None else RetryPolicy()
+        self.gzip = gzip
+        self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout, transport=transport)
+
+    def system_one(
+        self, state: State, questions: dict[str, Choice | Score | Noul], model: str | None = None,
+        exclude_from_training: bool = False,
+    ) -> SystemOneResponse:
+        request = SystemOneRequest(model=model, state=state, questions=questions,
+                                   exclude_from_training=exclude_from_training)
+        # The flag goes only when set, so every other body is exactly System One's.
+        body = request.model_dump(mode="json", exclude=None if exclude_from_training else {"exclude_from_training"})
+        return SystemOneResponse.model_validate(self._send("POST", "/v1/systemone", body).json())
+
+    def predict(self, rows: list[dict], model: str | None = None) -> list[dict]:
+        """Every row answered in order: `{"id"?, "answers": {id: Answer}}`, or `{"id"?, "error": {...}}` for
+        a row the server refused on its own."""
+        body = self._send("POST", "/v1/predict", {"model": model, "rows": rows}).json()
+        return [{**entry, "answers": {qid: ANSWER.validate_python(a) for qid, a in entry["answers"].items()}}
+                if "answers" in entry else entry
+                for entry in body["answers"]]
+
+    def evaluate(self, rows: list[dict], model: str | None = None) -> dict:
+        """The labeled rows scored: the report `ruling eval` prints, with the answering model's name."""
+        return self._send("POST", "/v1/evaluations", {"model": model, "rows": rows}).json()
+
+    def usage(self, starting_on: datetime | None = None, ending_before: datetime | None = None) -> list[dict]:
+        """Hourly buckets per model since the server started, within the window when one is given."""
+        params = {name: moment.isoformat() for name, moment in
+                  (("starting_on", starting_on), ("ending_before", ending_before)) if moment is not None}
+        return self._send("GET", "/v1/usage", params=params).json()["usage"]
+
+    def outcome(self, decision_id: str, labels: dict) -> dict:
+        """Report what turned out to be right for a logged decision; returns the labels as stored."""
+        return self._send("POST", f"/v1/decisions/{decision_id}/outcome", {"labels": labels}).json()["labels"]
+
+    def models(self) -> list[ModelInfo]:
+        return ModelList.model_validate(self._send("GET", "/v1/models").json()).models
+
+    def _send(self, method: str, path: str, body: dict | None = None, params: dict | None = None) -> httpx.Response:
+        content, headers = self._encode(body)
+        for attempt in range(self.retry.max_retries + 1):
+            last = attempt == self.retry.max_retries
+            try:
+                response = self._http.request(method, path, content=content, headers=headers, params=params)
+            except httpx.TransportError:
+                if last:
+                    raise
+                sleep(self.retry.delay(attempt, None))
+                continue
+            if response.status_code in RETRIED_STATUSES and not last:
+                sleep(self.retry.delay(attempt, response.headers.get("retry-after")))
+                continue
+            if response.is_error:
+                raise RulingError.from_response(response)
+            return response
+
+    def _encode(self, body: dict | None) -> tuple[bytes | None, dict[str, str]]:
+        if body is None:
+            return None, {}
+        content = json.dumps(body).encode()
+        headers = {"content-type": "application/json"}
+        if self.gzip and len(content) >= GZIP_THRESHOLD:
+            content = gzip.compress(content)
+            headers["content-encoding"] = "gzip"
+        return content, headers
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "RulingClient":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()

{ruling-0.2.0 → ruling-0.3.0}/ruling/server.py

@@ -1,24 +1,37 @@
-"""HTTP surface: one decision endpoint over a registry of models, model listing, health, and a playground page."""
+"""HTTP surface: decisions one at a time or in batches over a registry of models, scored evaluations,
+usage counts, model listing, health, and a playground page."""
 
 from __future__ import annotations
 
+import gzip
 import logging
+from importlib.metadata import version
 import math
 import threading
 import time
+import zlib
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 from fastapi import Depends, FastAPI, Request
 from fastapi.concurrency import run_in_threadpool
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from pydantic import BaseModel, Field, ValidationError
+from starlette.datastructures import Headers
+from starlette.types import ASGIApp, Receive, Scope, Send
 
 from ruling.config import Settings
 from ruling.engine import InputTooLong
+from ruling.evals import Record, as_record, collect, predict, report
 from ruling.hosted import HostUnavailable
 from ruling.ledger import DecisionLedger, Outcome, UnknownDecision
 from ruling.questions import (
+    MAX_PREDICT_ROWS,
     InvalidOptions,
     ModelList,
     SystemOneRequest,
@@ -29,6 +42,7 @@ from ruling.questions import (
 )
 from ruling.registry import Registry, UnknownModel
 from ruling.retrain import CURRENT, history
+from ruling.usage import Metered, UsageMeter
 
 log = logging.getLogger("ruling")
 PLAYGROUND = Path(__file__).with_name("playground.html")
@@ -86,12 +100,145 @@ class Admission:
             estimate = self.pending * (self.seconds_per_request or 0.0)
         return max(1, math.ceil(estimate))
 
+    @contextmanager
+    def slot(self) -> Iterator[Callable[[], float]]:
+        """Hold one place in the queue for the block, or refuse with a wait; yields the elapsed time so far."""
+        if not self.enter():
+            raise ApiError(429, "queue_full", f"{self.limit} requests are already in flight",
+                           retry_after=self.retry_after())
+        started = time.perf_counter()
+
+        def elapsed() -> float:
+            return time.perf_counter() - started
+
+        try:
+            yield elapsed
+        finally:
+            self.leave(elapsed())
+
+
+class GzipRequests:
+    """Inflates a request body sent with `Content-Encoding: gzip`, so a client can shrink a large batch."""
+
+    def __init__(self, app: ASGIApp):
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http" or Headers(scope=scope).get("content-encoding") != "gzip":
+            await self.app(scope, receive, send)
+            return
+        packed = b""
+        while True:
+            message = await receive()
+            packed += message.get("body", b"")
+            if not message.get("more_body"):
+                break
+        try:
+            body = gzip.decompress(packed)
+        except (gzip.BadGzipFile, EOFError, zlib.error) as exc:
+            await ApiError(422, "invalid_input", f"the body is not valid gzip: {exc}").response()(scope, receive, send)
+            return
+        headers = [(k, v) for k, v in scope["headers"] if k not in (b"content-encoding", b"content-length")]
+        headers.append((b"content-length", str(len(body)).encode()))
+
+        async def receive_inflated() -> dict:
+            return {"type": "http.request", "body": body, "more_body": False}
+
+        await self.app({**scope, "headers": headers}, receive_inflated, send)
+
+
+class BatchRequest(BaseModel):
+    """Rows in any shape `ruling.evals` reads for `ruling predict`; a row's shape is checked row by row."""
+
+    model: str | None = None
+    rows: list[Any] = Field(min_length=1, max_length=MAX_PREDICT_ROWS)
+
+
+def parse_row(row: Any) -> Record:
+    if not isinstance(row, dict):
+        raise ValueError("a row must be an object")
+    return Record.model_validate(as_record(row))
+
+
+def row_error(exc: ValueError) -> dict[str, str]:
+    """The API's code for what refused a row. A schema failure is reported by its first problem; one a
+    record's own validator raised, such as a one-option choice, keeps the code that error carries."""
+    if isinstance(exc, ValidationError):
+        first = exc.errors()[0]
+        cause = first.get("ctx", {}).get("error")
+        if isinstance(cause, ValueError):
+            return row_error(cause)
+        where = ".".join(str(part) for part in first["loc"])
+        return {"code": "invalid_input", "message": f"{where}: {first['msg']}" if where else first["msg"]}
+    if isinstance(exc, InputTooLong):
+        return {"code": "context_too_long", "message": str(exc)}
+    if isinstance(exc, InvalidOptions):
+        return {"code": "invalid_options", "message": str(exc)}
+    if isinstance(exc, UnsupportedModality):
+        return {"code": "unsupported_modality", "message": str(exc)}
+    return {"code": "invalid_input", "message": str(exc)}
+
+
+def answer_rows(engine: Metered, rows: list[Any]) -> list[dict]:
+    """One entry per row in order: its answers, or the error that refused that row alone."""
+    entries: list[dict | None] = [None] * len(rows)
+    records, places = [], []
+    for index, row in enumerate(rows):
+        try:
+            records.append(parse_row(row))
+            places.append(index)
+        except ValueError as exc:
+            named = isinstance(row, dict) and row.get("id") is not None
+            entries[index] = {**({"id": row["id"]} if named else {}), "error": row_error(exc)}
+    for index, line in zip(places, predict(engine, records)):
+        entries[index] = line
+    return entries
+
+
+def labeled_records(engine, rows: list[Any]) -> list[Record]:
+    """Every row as a record the engine can score, or the 422 naming the first row it cannot."""
+    records = []
+    for index, row in enumerate(rows):
+        try:
+            record = parse_row(row)
+            check_askable(record.questions, engine.max_options)
+            check_readable(record.state, engine.reads_images)
+        except ValueError as exc:
+            raise ApiError(422, param=f"rows.{index}", **row_error(exc)) from exc
+        records.append(record)
+    if not any(record.labels for record in records):
+        raise ApiError(422, "invalid_input", "no row carries labels; nothing to score against", "rows")
+    return records
+
 
 def create_app(registry: Registry, api_key: str | None, max_pending: int,
-               ledger: DecisionLedger | None = None) -> FastAPI:
-    app = FastAPI(title="ruling", version="0.2.0")
+               ledger: DecisionLedger | None = None, usage: UsageMeter | None = None) -> FastAPI:
+    app = FastAPI(title="ruling", version=version("ruling"))
+    app.add_middleware(GzipRequests)
     bearer = HTTPBearer(auto_error=False)
     admission = Admission(max_pending)
+    usage = usage if usage is not None else UsageMeter()
+
+    def resolve(model: str | None):
+        try:
+            return registry.resolve(model)
+        except UnknownModel as exc:
+            raise ApiError(404, "unknown_model", str(exc), "model") from exc
+
+    @contextmanager
+    def engine_errors() -> Iterator[None]:
+        """What an engine raises while answering, in the API's shape."""
+        try:
+            yield
+        except InputTooLong as exc:
+            param = f"questions.{exc.question_id}" if exc.question_id is not None else None
+            raise ApiError(422, "context_too_long", str(exc), param) from exc
+        except InvalidOptions as exc:
+            raise ApiError(422, "invalid_options", str(exc), f"questions.{exc.question_id}") from exc
+        except ValueError as exc:
+            raise ApiError(422, "invalid_input", str(exc)) from exc
+        except HostUnavailable as exc:
+            raise ApiError(503, "upstream_unavailable", str(exc)) from exc
 
     def authorize(credentials: HTTPAuthorizationCredentials | None = Depends(bearer)) -> None:
         if api_key is None:
@@ -113,10 +260,7 @@ def create_app(registry: Registry, api_key: str | None, max_pending: int,
     @app.post("/v1/systemone", response_model=SystemOneResponse, response_model_exclude_none=True,
               dependencies=[Depends(authorize)])
     async def system_one(request: SystemOneRequest) -> SystemOneResponse:
-        try:
-            name, engine = registry.resolve(request.model)
-        except UnknownModel as exc:
-            raise ApiError(404, "unknown_model", str(exc), "model") from exc
+        name, engine = resolve(request.model)
         try:
             check_askable(request.questions, engine.max_options)
         except InvalidOptions as exc:
@@ -125,26 +269,11 @@ def create_app(registry: Registry, api_key: str | None, max_pending: int,
             check_readable(request.state, engine.reads_images)
         except UnsupportedModality as exc:
             raise ApiError(422, "unsupported_modality", f"{name}: {exc}", "state") from exc
-        if not admission.enter():
-            raise ApiError(429, "queue_full", f"{admission.limit} requests are already in flight",
-                           retry_after=admission.retry_after())
-        started = time.perf_counter()
-        try:
+        with admission.slot() as elapsed, engine_errors():
             response = await run_in_threadpool(engine.evaluate, request)
-        except InputTooLong as exc:
-            param = f"questions.{exc.question_id}" if exc.question_id is not None else None
-            raise ApiError(422, "context_too_long", str(exc), param) from exc
-        except InvalidOptions as exc:
-            raise ApiError(422, "invalid_options", str(exc), f"questions.{exc.question_id}") from exc
-        except ValueError as exc:
-            raise ApiError(422, "invalid_input", str(exc)) from exc
-        except HostUnavailable as exc:
-            raise ApiError(503, "upstream_unavailable", str(exc)) from exc
-        finally:
-            elapsed = time.perf_counter() - started
-            admission.leave(elapsed)
+        usage.record(name, 1, response.usage.input_tokens)
         log.info("systemone model=%s questions=%d input_tokens=%d ms=%.0f",
-                 name, len(request.questions), response.usage.input_tokens, elapsed * 1000)
+                 name, len(request.questions), response.usage.input_tokens, elapsed() * 1000)
         # An entry answers under its own name; the primary keeps the engine's report, which a
         # failover changes to say which side answered.
         if name != registry.default:
@@ -153,6 +282,38 @@ def create_app(registry: Registry, api_key: str | None, max_pending: int,
             response = response.model_copy(update={"id": await run_in_threadpool(ledger.record, request, response)})
         return response
 
+    @app.post("/v1/predict", dependencies=[Depends(authorize)])
+    async def predict_rows(request: BatchRequest) -> dict:
+        """Every row answered in order; a row the model refuses carries its error and the batch goes on."""
+        name, engine = resolve(request.model)
+        metered = Metered(engine)
+        with admission.slot() as elapsed, engine_errors():
+            entries = await run_in_threadpool(answer_rows, metered, request.rows)
+        answered = sum("answers" in entry for entry in entries)
+        usage.record(name, answered, metered.input_tokens)
+        log.info("predict model=%s rows=%d answered=%d input_tokens=%d ms=%.0f",
+                 name, len(entries), answered, metered.input_tokens, elapsed() * 1000)
+        return {"model": name, "answers": entries, "usage": {"input_tokens": metered.input_tokens, "output_tokens": 0}}
+
+    @app.post("/v1/evaluations", dependencies=[Depends(authorize)])
+    async def evaluations(request: BatchRequest) -> dict:
+        """The rows scored against their labels: exactly the report `ruling eval` prints."""
+        name, engine = resolve(request.model)
+        metered = Metered(engine)
+        with admission.slot() as elapsed, engine_errors():
+            records = await run_in_threadpool(labeled_records, engine, request.rows)
+            observations = await run_in_threadpool(collect, metered, records)
+        scored = len({o.record for o in observations if o.logits is not None})
+        usage.record(name, scored, metered.input_tokens)
+        log.info("evaluations model=%s rows=%d scored=%d input_tokens=%d ms=%.0f",
+                 name, len(records), scored, metered.input_tokens, elapsed() * 1000)
+        return {"model": name, **report(observations, engine.calibration)}
+
+    @app.get("/v1/usage", dependencies=[Depends(authorize)])
+    def usage_report(starting_on: datetime | None = None, ending_before: datetime | None = None) -> dict:
+        """Hourly counts per model since the server started, kept in memory only."""
+        return {"usage": usage.buckets(starting_on, ending_before)}
+
     if ledger is not None:
         @app.post("/v1/decisions/{decision}/outcome", dependencies=[Depends(authorize)])
         async def outcome(decision: str, body: Outcome) -> dict:

{ruling-0.2.0 → ruling-0.3.0}/ruling/train.py

@@ -85,6 +85,7 @@ class TrainConfig:
 
 
 CHECKPOINT = "checkpoint"
+TRAINING = "training.json"
 # Rendering happens in chunks of this many batches, which are then sorted by length and cut into batches.
 CHUNK_BATCHES = 64
 
@@ -181,6 +182,36 @@ def config_mismatch(saved: dict, requested: dict) -> list[str]:
     return sorted(k for k in set(saved) | set(requested) if k != "steps" and saved.get(k) != requested.get(k))
 
 
+def progress(step: int, steps: int, elapsed: float) -> dict:
+    """How far a run is and when it should end, rating the steps left at the run's pace so far.
+
+    `elapsed` covers every step since the run began, across resumes, so the pace is the whole run's."""
+    return {"elapsed_seconds": elapsed, "progress": step / steps, "eta_seconds": (steps - step) * elapsed / step}
+
+
+class WandbSink:
+    """Mirrors every history entry to a Weights & Biases run, opened on entering and finished on leaving,
+    marked failed when training raised. The package is imported here only, so it stays optional."""
+
+    def __init__(self, project: str | None, config: TrainConfig):
+        try:
+            import wandb
+        except ImportError as error:
+            raise ImportError("Weights & Biases logging needs the wandb package: pip install 'ruling[wandb]'") from error
+        self.wandb, self.project, self.config = wandb, project, config
+        self.run = None
+
+    def __enter__(self) -> "WandbSink":
+        self.run = self.wandb.init(project=self.project, config=config_json(self.config))
+        return self
+
+    def __call__(self, entry: dict) -> None:
+        self.run.log(entry, step=entry["step"])
+
+    def __exit__(self, kind, value, traceback) -> None:
+        self.run.finish(exit_code=0 if kind is None else 1)
+
+
 def save_checkpoint(directory: Path, weights: dict[str, mx.array], optimizer_state: dict, state: dict) -> None:
     """Everything a run needs to continue from its next step: LoRA weights, optimizer state and the trainer's bookkeeping."""
     directory.mkdir(parents=True, exist_ok=True)
@@ -418,9 +449,10 @@ class Trainer:
         meta = {"config": {**config_json(self.config), "data": data},
                 "base_revision": self.engine.revision, "train_records": len(self.train_records),
                 "train_examples": len(self.train_examples), "valid_examples": len(self.valid_examples), "history": history}
-        (directory / "training.json").write_text(json.dumps(meta, indent=2) + "\n")
+        (directory / TRAINING).write_text(json.dumps(meta, indent=2) + "\n")
 
-    def run(self, log=print) -> list[dict]:
+    def run(self, log=print, sink=None) -> list[dict]:
+        """Train to `steps`, returning the history; `sink` receives each history entry as it is made."""
         config = self.config
         model = self.engine.model
         model.train()
@@ -434,10 +466,16 @@ class Trainer:
         trainable = sum(v.size for _, v in tree_flatten(model.trainable_parameters()))
         log(f"trainable parameters {trainable:,}; {len(self.train_examples):,} train and {len(self.valid_examples):,} validation examples")
 
+        def record(entry: dict) -> None:
+            history.append(entry)
+            log(json.dumps(entry))
+            if sink is not None:
+                sink(entry)
+
         batches = self._batches()
         if self.checkpoint is None:
-            history = [{"step": 0, **self.validate()}]
-            log(json.dumps(history[-1]))
+            history = []
+            record({"step": 0, **self.validate()})
             self.save(config.output, history)
             first, tokens_seen, elapsed = 1, 0, 0.0
         else:
@@ -464,17 +502,16 @@ class Trainer:
             if step % config.eval_every == 0 or step == config.steps:
                 elapsed = time.perf_counter() - started
                 entry = {"step": step, "train_loss": float(np.mean(window)), "tokens_per_second": tokens_seen / elapsed,
-                         **self.validate()}
+                         **progress(step, config.steps, elapsed), **self.validate()}
                 window = []
-                history.append(entry)
-                log(json.dumps(entry))
+                record(entry)
                 if entry["loss"] < best:
                     best = entry["loss"]
                     self.save(config.output, history)
                 else:
-                    meta = json.loads((config.output / "training.json").read_text())
+                    meta = json.loads((config.output / TRAINING).read_text())
                     meta["history"] = history
-                    (config.output / "training.json").write_text(json.dumps(meta, indent=2) + "\n")
+                    (config.output / TRAINING).write_text(json.dumps(meta, indent=2) + "\n")
                 save_checkpoint(config.output / CHECKPOINT, dict(tree_flatten(model.trainable_parameters())), optimizer.state,
                                 {"step": step, "tokens_seen": tokens_seen, "elapsed": elapsed, "stream": batches.state(),
                                  "history": history, "config": config_json(config)})

ruling-0.3.0/ruling/usage.py

@@ -0,0 +1,75 @@
+"""What a server answered, counted per hour and per model.
+
+The counts live in memory for the server's lifetime: a restart starts them
+over, and nothing is written to disk. One decision is one `/v1/systemone`
+request or one row of a `/v1/predict` or `/v1/evaluations` batch that was
+answered; refused requests and rows count nothing.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+HOUR = 3600
+
+
+@dataclass
+class _Counts:
+    decisions: int = 0
+    input_tokens: int = 0
+
+
+class UsageMeter:
+    def __init__(self, now: Callable[[], float] = time.time):
+        self._now = now
+        self._buckets: dict[tuple[int, str], _Counts] = {}
+        self._lock = threading.Lock()
+
+    def record(self, model: str, decisions: int, input_tokens: int) -> None:
+        hour = int(self._now() // HOUR) * HOUR
+        with self._lock:
+            counts = self._buckets.setdefault((hour, model), _Counts())
+            counts.decisions += decisions
+            counts.input_tokens += input_tokens
+
+    def buckets(self, starting_on: datetime | None = None, ending_before: datetime | None = None) -> list[dict]:
+        """Every bucket whose hour starts within the window, by hour then model; a naive bound is read as UTC."""
+        start, end = _epoch(starting_on), _epoch(ending_before)
+        with self._lock:
+            items = sorted(self._buckets.items())
+        return [{"hour": datetime.fromtimestamp(hour, UTC).isoformat(), "model": model,
+                 "decisions": counts.decisions, "input_tokens": counts.input_tokens}
+                for (hour, model), counts in items
+                if (start is None or hour >= start) and (end is None or hour < end)]
+
+
+class Metered:
+    """An engine whose scoring passes add up their input tokens, so a batch that answers
+    through `ruling.evals` can report what it cost without the evaluation code counting."""
+
+    def __init__(self, engine):
+        self.engine = engine
+        self.input_tokens = 0
+
+    def __getattr__(self, name: str):
+        return getattr(self.engine, name)
+
+    def score(self, state, questions):
+        scored = self.engine.score(state, questions)
+        self.input_tokens += scored.input_tokens
+        return scored
+
+    def evaluate(self, request):
+        response = self.engine.evaluate(request)
+        self.input_tokens += response.usage.input_tokens
+        return response
+
+
+def _epoch(moment: datetime | None) -> float | None:
+    if moment is None:
+        return None
+    return (moment if moment.tzinfo is not None else moment.replace(tzinfo=UTC)).timestamp()

ruling-0.2.0/ruling/sdk.py

@@ -1,52 +0,0 @@
-"""Python client mirroring the shape of the hosted System One SDK."""
-
-from __future__ import annotations
-
-import os
-
-import httpx
-
-from ruling.questions import Choice, ModelInfo, ModelList, Noul, Score, State, SystemOneRequest, SystemOneResponse
-
-DEFAULT_BASE_URL = "http://127.0.0.1:8010"
-
-
-class RulingClient:
-    def __init__(self, base_url: str | None = None, api_key: str | None = None, timeout: float = 60.0):
-        self.base_url = base_url or os.environ.get("RULING_BASE_URL", DEFAULT_BASE_URL)
-        # The server's own RULING_API_KEY, so one environment configures both ends.
-        api_key = api_key or os.environ.get("RULING_API_KEY")
-        headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
-        self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout)
-
-    def system_one(
-        self, state: State, questions: dict[str, Choice | Score | Noul], model: str | None = None,
-        exclude_from_training: bool = False,
-    ) -> SystemOneResponse:
-        request = SystemOneRequest(model=model, state=state, questions=questions,
-                                   exclude_from_training=exclude_from_training)
-        # The flag goes only when set, so every other body is exactly System One's.
-        body = request.model_dump(exclude=None if exclude_from_training else {"exclude_from_training"})
-        response = self._http.post("/v1/systemone", json=body)
-        response.raise_for_status()
-        return SystemOneResponse.model_validate(response.json())
-
-    def outcome(self, decision_id: str, labels: dict) -> dict:
-        """Report what turned out to be right for a logged decision; returns the labels as stored."""
-        response = self._http.post(f"/v1/decisions/{decision_id}/outcome", json={"labels": labels})
-        response.raise_for_status()
-        return response.json()["labels"]
-
-    def models(self) -> list[ModelInfo]:
-        response = self._http.get("/v1/models")
-        response.raise_for_status()
-        return ModelList.model_validate(response.json()).models
-
-    def close(self) -> None:
-        self._http.close()
-
-    def __enter__(self) -> "RulingClient":
-        return self
-
-    def __exit__(self, *exc) -> None:
-        self.close()