ruling 0.2.0__py3-none-any.whl

ruling/questions.py

@@ -0,0 +1,280 @@
+"""Wire types for the System One API: questions in, typed answers out."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+from typing import Annotated, Any, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+State = Union[str, dict[str, Any], list[Any]]
+
+# Instructions, option descriptions, and level descriptions may be plain text or
+# JSON structure; structure is rendered compactly into the prompt.
+Text = Union[str, dict[str, Any], list[Any]]
+
+# The hosted services' request limits, so a client written against them never
+# sees a different answer here. Above the tokenizer's single-token codes a
+# Choice is answered in two stages (see Engine).
+MAX_CHOICE_OPTIONS = 255
+MAX_SCORE_LEVELS = 10
+MAX_QUESTIONS = 64
+
+# An image is {"type": "image", "media_type", "data": <base64>} anywhere in the state,
+# PNG, JPEG or WebP, at most 8 per decision and 5 MB each.
+MAX_IMAGES = 8
+MAX_IMAGE_BYTES = 5 * 2**20
+IMAGE_REFERENCE = "<image {}>"
+
+# The hosted SDK sends its default model name on every request. Treating these as
+# "whichever model this server loaded" is what lets that client talk to ruling
+# unchanged; the response still reports the model that actually answered.
+DEFAULT_MODEL_ALIASES = ("jev-latest", "default")
+
+
+def render_text(value: Text) -> str:
+    return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+
+
+def has_text(value: Text | None) -> bool:
+    return value is not None and bool(render_text(value).strip())
+
+
+class Strict(BaseModel):
+    """A question refuses fields it does not know, as the hosted services do, so a typo
+    such as `instruction` fails loudly instead of asking a different question."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class Choice(Strict):
+    """Pick one option from a closed set. Criteria maps option key to an optional description.
+
+    The schema admits one option, as the hosted services' does; `check_askable` is what
+    refuses it, naming the question.
+    """
+
+    type: Literal["choice"] = "choice"
+    instructions: Text | None = None
+    criteria: dict[str, Text | None] = Field(min_length=1, max_length=MAX_CHOICE_OPTIONS)
+
+
+class Score(Strict):
+    """Place the state on an ordered scale. Criteria lists the levels from lowest to highest."""
+
+    type: Literal["score"] = "score"
+    instructions: Text | None = None
+    criteria: list[Text] = Field(min_length=2, max_length=MAX_SCORE_LEVELS)
+
+    @field_validator("criteria")
+    @classmethod
+    def levels_are_distinct(cls, levels: list[Text]) -> list[Text]:
+        rendered = [render_text(level) for level in levels]
+        if len(set(rendered)) != len(rendered):
+            raise ValueError("score levels must be distinct")
+        return levels
+
+
+class NoulCriteria(Strict):
+    """Optional descriptions of what a true and a false answer mean."""
+
+    true: Text | None = None
+    false: Text | None = None
+
+
+class Noul(Strict):
+    """Judge whether a statement about the state is true. The statement is the instructions,
+    or, without them, what the criteria say a true or a false answer means."""
+
+    type: Literal["noul"] = "noul"
+    instructions: Text | None = None
+    criteria: NoulCriteria | None = None
+
+    def described(self) -> bool:
+        return self.criteria is not None and (has_text(self.criteria.true) or has_text(self.criteria.false))
+
+
+Question = Annotated[Union[Choice, Score, Noul], Field(discriminator="type")]
+
+
+# What a noul asks when it has no instructions and only its criteria describe the answers.
+DESCRIBED_NOUL = "Which description fits the state?"
+
+
+def question_statement(question: Question) -> str:
+    """The question's instructions as prompt text, or the standing question a described noul asks."""
+    return render_text(question.instructions) if has_text(question.instructions) else DESCRIBED_NOUL
+
+
+class Image(Strict):
+    type: Literal["image"]
+    media_type: Literal["image/png", "image/jpeg", "image/webp"]
+    data: str
+
+    @field_validator("data")
+    @classmethod
+    def decodes_within_the_limit(cls, data: str) -> str:
+        try:
+            size = len(base64.b64decode(data, validate=True))
+        except binascii.Error as exc:
+            raise ValueError(f"image data must be base64: {exc}") from exc
+        if size > MAX_IMAGE_BYTES:
+            raise ValueError(f"an image may be at most {MAX_IMAGE_BYTES // 2**20} MB, got {size / 2**20:.1f} MB")
+        return data
+
+    def data_url(self) -> str:
+        return f"data:{self.media_type};base64,{self.data}"
+
+
+def _is_image(value: Any) -> bool:
+    return isinstance(value, dict) and value.get("type") == "image"
+
+
+def state_images(state: State) -> list[Image]:
+    """Every image in the state, depth first in document order, each validated."""
+    if _is_image(state):
+        return [Image.model_validate(state)]
+    children = state.values() if isinstance(state, dict) else state if isinstance(state, list) else ()
+    return [image for child in children for image in state_images(child)]
+
+
+def without_images(state: State) -> State:
+    """The state with each image replaced by its reference, `<image 1>` and on, in `state_images` order."""
+    counter = iter(range(1, MAX_IMAGES + 1))
+
+    def replace(value: Any) -> Any:
+        if _is_image(value):
+            return IMAGE_REFERENCE.format(next(counter))
+        if isinstance(value, dict):
+            return {key: replace(child) for key, child in value.items()}
+        if isinstance(value, list):
+            return [replace(child) for child in value]
+        return value
+
+    return replace(state)
+
+
+class UnsupportedModality(ValueError):
+    """A state holds images and the model that would answer reads only text."""
+
+
+def check_readable(state: State, reads_images: bool) -> None:
+    if not reads_images and state_images(state):
+        raise UnsupportedModality("this model reads only text; the state holds images")
+
+
+class InvalidOptions(ValueError):
+    """A well-formed question the model cannot answer, named so the error can point at it."""
+
+    def __init__(self, question_id: str, message: str):
+        super().__init__(f"questions.{question_id}: {message}")
+        self.question_id = question_id
+
+
+def check_askable(questions: dict[str, Question], max_options: int) -> None:
+    """Refuse what the hosted services refuse after schema validation: a choice outside
+    2..`max_options`, or a question with nothing to ask."""
+    for qid, question in questions.items():
+        if isinstance(question, Choice) and not 2 <= len(question.criteria) <= max_options:
+            raise InvalidOptions(qid, f"a choice needs 2 to {max_options} options, got {len(question.criteria)}")
+        if not has_text(question.instructions) and not (isinstance(question, Noul) and question.described()):
+            raise InvalidOptions(qid, "needs instructions (a yes/no question may describe true or false instead)")
+
+
+class SystemOneRequest(BaseModel):
+    model: str | None = None
+    state: State
+    questions: dict[str, Question] = Field(min_length=1, max_length=MAX_QUESTIONS)
+    # Marks traffic that must never become training data, such as benchmarks and evaluations.
+    exclude_from_training: bool = False
+
+    @field_validator("state")
+    @classmethod
+    def images_are_valid(cls, state: State) -> State:
+        return check_images(state)
+
+    @model_validator(mode="after")
+    def ids_are_non_empty(self) -> "SystemOneRequest":
+        if any(not qid for qid in self.questions):
+            raise ValueError("question ids must be non-empty")
+        return self
+
+
+def check_images(state: State) -> State:
+    """Validate every image in the state and their count; returns the state unchanged."""
+    images = state_images(state)
+    if len(images) > MAX_IMAGES:
+        raise ValueError(f"a state may hold at most {MAX_IMAGES} images, got {len(images)}")
+    return state
+
+
+class ChoiceAnswer(BaseModel):
+    type: Literal["choice"] = "choice"
+    choice: str
+    probabilities: dict[str, float]
+    confidence: float
+
+
+class ScoreAnswer(BaseModel):
+    type: Literal["score"] = "score"
+    score: float
+    legend: dict[str, str]
+    probabilities: dict[str, float]
+    confidence: float
+
+
+class NoulAnswer(BaseModel):
+    type: Literal["noul"] = "noul"
+    noul: float
+
+
+Answer = Annotated[Union[ChoiceAnswer, ScoreAnswer, NoulAnswer], Field(discriminator="type")]
+
+
+class Usage(BaseModel):
+    input_tokens: int
+    output_tokens: int = 0
+
+
+class SystemOneResponse(BaseModel):
+    id: str | None = None
+    """The decision's id in the server's ledger, for reporting its outcome; absent when nothing is logged."""
+    model: str
+    answers: dict[str, Answer]
+    usage: Usage
+
+
+class ModelInfo(BaseModel):
+    """One served model. `name`, `description` and `release_date` are what the hosted
+    API reports and what its SDK decodes; the rest is detail only a local engine has.
+    `name` is what a request sends as `model`; `base` is the weights behind it."""
+
+    name: str
+    description: str
+    release_date: str
+    id: str
+    base: str
+    revision: str | None
+    adapter: str | None
+    calibration: dict[str, float]
+    max_input_tokens: int
+    max_branch_tokens: int
+    max_options: int
+    rotations: int
+    prior_debias: bool
+    modalities: list[Literal["text", "image"]]
+
+
+class ModelList(BaseModel):
+    models: list[ModelInfo]
+
+
+def option_keys(question: Choice | Score | Noul) -> list[str]:
+    """The ordered answer set a question is scored over."""
+    if isinstance(question, Choice):
+        return list(question.criteria)
+    if isinstance(question, Score):
+        return [str(i) for i in range(len(question.criteria))]
+    return ["yes", "no"]

ruling/registry.py

@@ -0,0 +1,195 @@
+"""Which models one server answers for: the environment's primary, plus named entries from a file.
+
+Entries that fine-tune the same base share its loaded weights: the LoRA layers
+are swapped under the engine lock before each request, so a server can hold a
+plain base, a tuned adapter per decision, and a large escalation model at the
+cost of one copy of each base.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from ruling.calibration import Calibration
+from ruling.cascade import CascadeEngine
+from ruling.config import Settings
+from ruling.decision import DecisionEngine, is_checkpoint
+from ruling.engine import Engine, model_released
+from ruling.failover import FailoverEngine
+from ruling.hosted import (
+    HOSTED_PREFIX,
+    OPENROUTER_PREFIX,
+    TYPESAFE_PREFIX,
+    VLLM_PREFIX,
+    Budget,
+    HostedEngine,
+    OpenRouterEngine,
+    TypeSafeEngine,
+    VllmEngine,
+)
+from ruling.questions import DEFAULT_MODEL_ALIASES, ModelInfo
+from ruling.vision import VISION_PREFIX
+
+ENTRY_FIELDS = ("model", "adapter", "calibration", "base_url")
+REMOTE_PREFIXES = (TYPESAFE_PREFIX, VLLM_PREFIX, HOSTED_PREFIX)
+
+
+class UnknownModel(KeyError):
+    def __init__(self, name: str, known: list[str]):
+        super().__init__(name)
+        self.name = name
+        self.known = known
+
+    def __str__(self) -> str:
+        return f"no model {self.name!r}; this server answers for {', '.join(repr(k) for k in self.known)}"
+
+
+@dataclass(frozen=True)
+class Entry:
+    model: str
+    adapter: Path | None = None
+    calibration: Path | None = None
+    base_url: str | None = None
+    """Where a `typesafe:`, `vllm:` or `openai:` entry is served, in place of the global setting."""
+
+    @classmethod
+    def parse(cls, name: str, raw: object, relative_to: Path) -> "Entry":
+        if not isinstance(raw, dict) or "model" not in raw:
+            raise ValueError(f"model {name!r} must be an object with at least a 'model' field")
+        unknown = set(raw) - set(ENTRY_FIELDS)
+        if unknown:
+            raise ValueError(f"model {name!r} has unknown fields {sorted(unknown)}; allowed: {list(ENTRY_FIELDS)}")
+        paths = {key: relative_to / raw[key] for key in ("adapter", "calibration") if raw.get(key)}
+        base_url = raw.get("base_url")
+        if base_url and not str(raw["model"]).startswith(REMOTE_PREFIXES):
+            raise ValueError(f"model {name!r} has a base_url, which only {', '.join(REMOTE_PREFIXES)} models use")
+        return cls(model=str(raw["model"]), base_url=base_url, **paths)
+
+
+def load_entries(path: Path) -> dict[str, Entry]:
+    """Named entries from a JSON object; relative paths resolve against the file's directory."""
+    data = json.loads(path.read_text())
+    if not isinstance(data, dict):
+        raise ValueError(f"{path} must hold a JSON object of model name to entry")
+    entries = {}
+    for name, raw in data.items():
+        if not name or name in DEFAULT_MODEL_ALIASES:
+            raise ValueError(f"{name!r} cannot name a model; it is empty or a default alias")
+        entries[name] = Entry.parse(name, raw, path.parent)
+    return entries
+
+
+def build_engine(settings: Settings):
+    """Any OpenAI-compatible host, OpenRouter, a trained decision checkpoint, or a local decoder.
+
+    With `cascade_to` set, that model answers whichever questions the first one is
+    unsure about, judged against `cascade_threshold`. With `failover_to` set, that
+    model answers whenever the first one is unavailable.
+    """
+    if settings.failover_to:
+        primary = build_engine(replace(settings, failover_to=None))
+        fallback = build_engine(replace(settings, model=settings.failover_to, failover_to=None, cascade_to=None,
+                                        adapter_path=None, calibration_path=None))
+        return FailoverEngine(primary, fallback)
+    if settings.cascade_to:
+        if settings.cascade_threshold is None:
+            raise ValueError("set RULING_CASCADE_THRESHOLD, the top probability below which a question is escalated")
+        first = build_engine(replace(settings, cascade_to=None))
+        second = build_engine(replace(settings, model=settings.cascade_to, cascade_to=None, adapter_path=None,
+                                      calibration_path=None))
+        return CascadeEngine(first, second, settings.cascade_threshold)
+    calibration = Calibration.load(settings.calibration_path) if settings.calibration_path else Calibration()
+    if settings.model.startswith(TYPESAFE_PREFIX):
+        return TypeSafeEngine(settings.model, calibration, settings.typesafe_base_url, settings.typesafe_api_key,
+                              settings.max_input_tokens, settings.max_branch_tokens)
+    if settings.model.startswith(VLLM_PREFIX):
+        if not settings.vllm_base_url:
+            raise ValueError("set RULING_VLLM_BASE_URL to the vLLM server, without a path")
+        return VllmEngine(settings.model, calibration, settings.vllm_base_url, settings.vllm_api_key,
+                          settings.max_input_tokens, settings.max_branch_tokens)
+    if settings.model.startswith(HOSTED_PREFIX):
+        if not settings.openai_base_url:
+            raise ValueError("set RULING_OPENAI_BASE_URL to the endpoint serving the model, ending in /v1")
+        return HostedEngine(settings.model, calibration, settings.rotations, settings.openai_base_url,
+                            settings.openai_api_key, concurrency=settings.openai_concurrency,
+                            top_logprobs=settings.openai_top_logprobs,
+                            extra_body=settings.openai_extra_body)
+    if settings.model.startswith(OPENROUTER_PREFIX):
+        if not settings.openrouter_providers:
+            raise ValueError("set RULING_OPENROUTER_PROVIDERS to the hosts allowed to serve the model")
+        return OpenRouterEngine(settings.model, calibration, settings.rotations, list(settings.openrouter_providers),
+                                Budget(settings.openrouter_budget_usd))
+    if settings.model.startswith(VISION_PREFIX):
+        from ruling.vision import VisionEngine
+        vision = VisionEngine(settings.model, Calibration() if settings.adapter_path else calibration,
+                              settings.max_input_tokens, settings.rotations, settings.prior_debias,
+                              settings.max_image_tokens, settings.max_branch_tokens)
+        # The adapter goes onto the loaded language half, as for any variant sharing a base.
+        return vision.variant(settings.adapter_path, calibration) if settings.adapter_path else vision
+    if is_checkpoint(settings.model):
+        return DecisionEngine(Path(settings.model), calibration, settings.max_input_tokens)
+    return Engine(settings.model, calibration, settings.max_input_tokens, settings.rotations, settings.prior_debias,
+                  settings.adapter_path, settings.max_branch_tokens)
+
+
+class Registry:
+    """The engines a server answers for, by the name a request sends as `model`."""
+
+    def __init__(self, default: str, engines: dict[str, object]):
+        if default not in engines:
+            raise ValueError(f"default model {default!r} is not among {sorted(engines)}")
+        self.default = default
+        self.engines = engines
+
+    @classmethod
+    def single(cls, engine) -> "Registry":
+        return cls(engine.model_id, {engine.model_id: engine})
+
+    @classmethod
+    def from_settings(cls, settings: Settings) -> "Registry":
+        primary = build_engine(settings)
+        engines: dict[str, object] = {primary.model_id: primary}
+        if settings.models_path:
+            for name, entry in load_entries(settings.models_path).items():
+                if name in engines:
+                    raise ValueError(f"model {name!r} is both the primary model and an entry in {settings.models_path}")
+                engines[name] = _build_entry(entry, settings, engines)
+        return cls(primary.model_id, engines)
+
+    def resolve(self, name: str | None) -> tuple[str, object]:
+        """The entry name and engine a request's `model` field selects; None and the aliases mean the default."""
+        if name is None or name in DEFAULT_MODEL_ALIASES:
+            name = self.default
+        try:
+            return name, self.engines[name]
+        except KeyError:
+            raise UnknownModel(name, sorted(self.engines)) from None
+
+    def info(self) -> list[ModelInfo]:
+        return [ModelInfo(
+            name=name, id=name, base=engine.model_id,
+            description=f"{engine.model_id} answered by ruling over {engine.rotations} answer orderings",
+            release_date=model_released(engine.model_id),
+            revision=engine.revision, adapter=engine.adapter, calibration=dict(engine.calibration.temperatures),
+            max_input_tokens=engine.max_input_tokens, max_branch_tokens=engine.max_branch_tokens,
+            max_options=engine.max_options, rotations=engine.rotations, prior_debias=engine.prior_debias,
+            modalities=["text", "image"] if engine.reads_images else ["text"],
+        ) for name, engine in self.engines.items()]
+
+
+def _build_entry(entry: Entry, settings: Settings, loaded: dict[str, object]):
+    """An engine for one entry, sharing an already loaded copy of its base when the LoRA shapes allow."""
+    calibration = Calibration.load(entry.calibration) if entry.calibration else Calibration()
+    for engine in loaded.values():
+        if isinstance(engine, Engine) and engine.model_id == entry.model:
+            try:
+                return engine.variant(entry.adapter, calibration)
+            except Engine.IncompatibleAdapter:
+                break
+    # Only the engine the model's prefix selects reads its address, so setting all three is exact.
+    address = ({"typesafe_base_url": entry.base_url, "vllm_base_url": entry.base_url, "openai_base_url": entry.base_url}
+               if entry.base_url else {})
+    return build_engine(replace(settings, model=entry.model, adapter_path=entry.adapter,
+                                calibration_path=entry.calibration, cascade_to=None, failover_to=None, **address))

ruling/retrain.py

@@ -0,0 +1,125 @@
+"""Learn from your own traffic: train on the decisions whose outcomes came back, and serve the result
+only when it is better.
+
+An adapters directory holds one version per run, `v1`, `v2`, …, and `current`,
+a relative link to the version being served (`RULING_ADAPTER=<dir>/current`).
+A run exports the ledger's resolved decisions, drops any whose state is in a
+held-out set, trains the next version (from `current`'s weights when there is
+one), scores both versions on the held-out sets, and repoints `current` only
+when the new one has lower log loss without losing accuracy. Every run leaves
+its records, its training log and the comparison in its version directory,
+promoted or not.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import replace
+from pathlib import Path
+
+import mlx.core as mx
+
+from ruling.calibration import Calibration
+from ruling.config import Settings
+from ruling.engine import ADAPTER_CONFIG
+from ruling.evals import Record, collect, decision_blocks, load_dataset
+from ruling.ledger import DecisionLedger
+from ruling.registry import build_engine
+from ruling.train import Trainer, TrainConfig, _state_key
+
+CURRENT = "current"
+REPORT = "retrain.json"
+DECISIONS = "decisions.jsonl"
+VERSION = re.compile(r"v(\d+)")
+
+
+def should_promote(current: dict, candidate: dict) -> tuple[bool, str]:
+    """Held-out log loss must fall and accuracy must not: a sharper but less often right adapter is not better."""
+    if candidate["nll"] >= current["nll"]:
+        return False, f"held-out log loss {candidate['nll']:.4f} is not below {current['nll']:.4f}"
+    if candidate["accuracy"] < current["accuracy"]:
+        return False, f"held-out accuracy {candidate['accuracy']:.4f} fell below {current['accuracy']:.4f}"
+    return True, (f"held-out log loss {current['nll']:.4f} -> {candidate['nll']:.4f}, "
+                  f"accuracy {current['accuracy']:.4f} -> {candidate['accuracy']:.4f}")
+
+
+def next_version(adapters: Path) -> Path:
+    numbers = [int(m.group(1)) for p in adapters.iterdir() if (m := VERSION.fullmatch(p.name))] if adapters.exists() else []
+    return adapters / f"v{max(numbers, default=0) + 1}"
+
+
+def promote(adapters: Path, version: Path) -> None:
+    """Point `current` at `version` in one rename, so a server starting meanwhile sees the old or the new, never neither."""
+    staged = adapters / f".{CURRENT}.next"
+    staged.unlink(missing_ok=True)
+    staged.symlink_to(version.name)
+    os.replace(staged, adapters / CURRENT)
+
+
+def trainable(records: list[Record], held_out: list[Record]) -> tuple[list[Record], int]:
+    """The records whose states appear in no held-out set, and how many were dropped for appearing in one."""
+    held = {_state_key(r.state) for r in held_out}
+    kept = [r for r in records if _state_key(r.state) not in held]
+    return kept, len(records) - len(kept)
+
+
+def retrain(ledger: DecisionLedger, adapters: Path, model: str, holdout: list[Path], settings: Settings,
+            min_records: int, train_options: dict, log=print) -> dict | None:
+    """One retraining run; returns its report, or None when too few decisions have outcomes yet."""
+    held_out = [r for path in holdout for r in load_dataset(path)]
+    records, dropped = trainable(list(ledger.records()), held_out)
+    if dropped:
+        log(f"dropped {dropped} decisions whose states are in the held-out sets")
+    if len(records) < min_records:
+        log(f"{len(records)} decisions have outcomes; waiting for {min_records}")
+        return None
+
+    current = adapters / CURRENT
+    start = current.resolve() if current.exists() else None
+    version = next_version(adapters)
+    version.mkdir(parents=True)
+    data = version / DECISIONS
+    data.write_text("".join(r.model_dump_json(exclude_none=True) + "\n" for r in records))
+    Trainer(TrainConfig(model=model, data=[data], output=version, holdout=holdout, from_adapter=start,
+                        max_image_tokens=settings.max_image_tokens, **_lora_layout(start), **train_options)).run(log)
+    # The trainer's copy of the model is unreachable now; return its memory before loading the scorer's.
+    mx.clear_cache()
+
+    served = build_engine(replace(settings, model=model, adapter_path=start, calibration_path=None, cascade_to=None,
+                                  failover_to=None))
+    candidate = served.variant(version, Calibration())
+    scores = {name: decision_blocks(collect(engine, held_out), Calibration())["overall"]
+              for name, engine in (("current", served), ("candidate", candidate))}
+    promoted, reason = should_promote(scores["current"], scores["candidate"])
+    report = {"version": version.name, "started_from": start.name if start else None, "records": len(records),
+              "dropped_held_out": dropped, "holdout": [str(p) for p in holdout], **scores,
+              "promoted": promoted, "reason": reason}
+    (version / REPORT).write_text(json.dumps(report, indent=2) + "\n")
+    if promoted:
+        promote(adapters, version)
+    log(f"{version.name}: {'promoted' if promoted else 'kept ' + (start.name if start else 'the base model')}; {reason}")
+    return report
+
+
+def history(adapters: Path) -> dict:
+    """Every finished run's report in version order, and the version `current` points at.
+
+    A version still training, or one whose run failed, has no report and is left out."""
+    link = adapters / CURRENT
+    reports = [json.loads((version / REPORT).read_text())
+               for version in sorted((p for p in adapters.iterdir() if VERSION.fullmatch(p.name)),
+                                     key=lambda p: int(p.name[1:]))
+               if (version / REPORT).exists()] if adapters.exists() else []
+    return {"current": link.resolve().name if link.exists() else None, "versions": reports}
+
+
+def _lora_layout(adapter: Path | None) -> dict:
+    """The LoRA settings to train with: the starting adapter's, which its weights only fit, else the trainer's defaults."""
+    if adapter is None:
+        return {}
+    saved = json.loads((adapter / ADAPTER_CONFIG).read_text())
+    lora = saved["lora_parameters"]
+    return {"lora_layers": saved["num_layers"], "lora_rank": lora["rank"], "lora_scale": lora["scale"],
+            "lora_dropout": lora["dropout"]}

ruling/sdk.py

@@ -0,0 +1,52 @@
+"""Python client mirroring the shape of the hosted System One SDK."""
+
+from __future__ import annotations
+
+import os
+
+import httpx
+
+from ruling.questions import Choice, ModelInfo, ModelList, Noul, Score, State, SystemOneRequest, SystemOneResponse
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8010"
+
+
+class RulingClient:
+    def __init__(self, base_url: str | None = None, api_key: str | None = None, timeout: float = 60.0):
+        self.base_url = base_url or os.environ.get("RULING_BASE_URL", DEFAULT_BASE_URL)
+        # The server's own RULING_API_KEY, so one environment configures both ends.
+        api_key = api_key or os.environ.get("RULING_API_KEY")
+        headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
+        self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout)
+
+    def system_one(
+        self, state: State, questions: dict[str, Choice | Score | Noul], model: str | None = None,
+        exclude_from_training: bool = False,
+    ) -> SystemOneResponse:
+        request = SystemOneRequest(model=model, state=state, questions=questions,
+                                   exclude_from_training=exclude_from_training)
+        # The flag goes only when set, so every other body is exactly System One's.
+        body = request.model_dump(exclude=None if exclude_from_training else {"exclude_from_training"})
+        response = self._http.post("/v1/systemone", json=body)
+        response.raise_for_status()
+        return SystemOneResponse.model_validate(response.json())
+
+    def outcome(self, decision_id: str, labels: dict) -> dict:
+        """Report what turned out to be right for a logged decision; returns the labels as stored."""
+        response = self._http.post(f"/v1/decisions/{decision_id}/outcome", json={"labels": labels})
+        response.raise_for_status()
+        return response.json()["labels"]
+
+    def models(self) -> list[ModelInfo]:
+        response = self._http.get("/v1/models")
+        response.raise_for_status()
+        return ModelList.model_validate(response.json()).models
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "RulingClient":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()