ruling 0.2.0__py3-none-any.whl

ruling/server.py

@@ -0,0 +1,198 @@
+"""HTTP surface: one decision endpoint over a registry of models, model listing, health, and a playground page."""
+
+from __future__ import annotations
+
+import logging
+import math
+import threading
+import time
+from pathlib import Path
+
+from fastapi import Depends, FastAPI, Request
+from fastapi.concurrency import run_in_threadpool
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+from ruling.config import Settings
+from ruling.engine import InputTooLong
+from ruling.hosted import HostUnavailable
+from ruling.ledger import DecisionLedger, Outcome, UnknownDecision
+from ruling.questions import (
+    InvalidOptions,
+    ModelList,
+    SystemOneRequest,
+    SystemOneResponse,
+    UnsupportedModality,
+    check_askable,
+    check_readable,
+)
+from ruling.registry import Registry, UnknownModel
+from ruling.retrain import CURRENT, history
+
+log = logging.getLogger("ruling")
+PLAYGROUND = Path(__file__).with_name("playground.html")
+HISTORY = Path(__file__).with_name("history.html")
+
+# Weight of the latest request in the running estimate of how long one takes, which
+# sizes the Retry-After a turned-away client is given.
+LATENCY_SMOOTHING = 0.2
+
+
+class ApiError(Exception):
+    """An error in the hosted API's shape: a stable `code` for programs, a `message` for people."""
+
+    def __init__(self, status: int, code: str, message: str, param: str | None = None, retry_after: int | None = None):
+        super().__init__(message)
+        self.status, self.code, self.message, self.param, self.retry_after = status, code, message, param, retry_after
+
+    def response(self) -> JSONResponse:
+        error = {"code": self.code, "message": self.message}
+        if self.param is not None:
+            error["param"] = self.param
+        headers = {"Retry-After": str(self.retry_after)} if self.retry_after is not None else None
+        return JSONResponse({"error": error}, status_code=self.status, headers=headers)
+
+
+class Admission:
+    """How many requests may be in flight at once, and how long a refused one should wait."""
+
+    def __init__(self, limit: int):
+        if limit < 1:
+            raise ValueError("at least one request must be admitted")
+        self.limit = limit
+        self.pending = 0
+        self.seconds_per_request: float | None = None
+        self._lock = threading.Lock()
+
+    def enter(self) -> bool:
+        with self._lock:
+            if self.pending >= self.limit:
+                return False
+            self.pending += 1
+            return True
+
+    def leave(self, elapsed: float) -> None:
+        with self._lock:
+            self.pending -= 1
+            if self.seconds_per_request is None:
+                self.seconds_per_request = elapsed
+            else:
+                self.seconds_per_request += LATENCY_SMOOTHING * (elapsed - self.seconds_per_request)
+
+    def retry_after(self) -> int:
+        """Whole seconds until the queue has likely drained, never less than one."""
+        with self._lock:
+            estimate = self.pending * (self.seconds_per_request or 0.0)
+        return max(1, math.ceil(estimate))
+
+
+def create_app(registry: Registry, api_key: str | None, max_pending: int,
+               ledger: DecisionLedger | None = None) -> FastAPI:
+    app = FastAPI(title="ruling", version="0.2.0")
+    bearer = HTTPBearer(auto_error=False)
+    admission = Admission(max_pending)
+
+    def authorize(credentials: HTTPAuthorizationCredentials | None = Depends(bearer)) -> None:
+        if api_key is None:
+            return
+        if credentials is None or credentials.credentials != api_key:
+            raise ApiError(401, "unauthorized", "invalid or missing bearer token")
+
+    @app.exception_handler(ApiError)
+    async def api_error(request: Request, exc: ApiError) -> JSONResponse:
+        return exc.response()
+
+    @app.exception_handler(RequestValidationError)
+    async def invalid_request(request: Request, exc: RequestValidationError) -> JSONResponse:
+        first, *rest = exc.errors()
+        param = ".".join(str(part) for part in first["loc"][1:]) or None
+        message = first["msg"] + (f" (and {len(rest)} more)" if rest else "")
+        return ApiError(422, "invalid_input", message, param).response()
+
+    @app.post("/v1/systemone", response_model=SystemOneResponse, response_model_exclude_none=True,
+              dependencies=[Depends(authorize)])
+    async def system_one(request: SystemOneRequest) -> SystemOneResponse:
+        try:
+            name, engine = registry.resolve(request.model)
+        except UnknownModel as exc:
+            raise ApiError(404, "unknown_model", str(exc), "model") from exc
+        try:
+            check_askable(request.questions, engine.max_options)
+        except InvalidOptions as exc:
+            raise ApiError(422, "invalid_options", str(exc), f"questions.{exc.question_id}") from exc
+        try:
+            check_readable(request.state, engine.reads_images)
+        except UnsupportedModality as exc:
+            raise ApiError(422, "unsupported_modality", f"{name}: {exc}", "state") from exc
+        if not admission.enter():
+            raise ApiError(429, "queue_full", f"{admission.limit} requests are already in flight",
+                           retry_after=admission.retry_after())
+        started = time.perf_counter()
+        try:
+            response = await run_in_threadpool(engine.evaluate, request)
+        except InputTooLong as exc:
+            param = f"questions.{exc.question_id}" if exc.question_id is not None else None
+            raise ApiError(422, "context_too_long", str(exc), param) from exc
+        except InvalidOptions as exc:
+            raise ApiError(422, "invalid_options", str(exc), f"questions.{exc.question_id}") from exc
+        except ValueError as exc:
+            raise ApiError(422, "invalid_input", str(exc)) from exc
+        except HostUnavailable as exc:
+            raise ApiError(503, "upstream_unavailable", str(exc)) from exc
+        finally:
+            elapsed = time.perf_counter() - started
+            admission.leave(elapsed)
+        log.info("systemone model=%s questions=%d input_tokens=%d ms=%.0f",
+                 name, len(request.questions), response.usage.input_tokens, elapsed * 1000)
+        # An entry answers under its own name; the primary keeps the engine's report, which a
+        # failover changes to say which side answered.
+        if name != registry.default:
+            response = response.model_copy(update={"model": name})
+        if ledger is not None and not request.exclude_from_training:
+            response = response.model_copy(update={"id": await run_in_threadpool(ledger.record, request, response)})
+        return response
+
+    if ledger is not None:
+        @app.post("/v1/decisions/{decision}/outcome", dependencies=[Depends(authorize)])
+        async def outcome(decision: str, body: Outcome) -> dict:
+            try:
+                record = await run_in_threadpool(ledger.outcome, decision, body.labels)
+            except UnknownDecision as exc:
+                raise ApiError(404, "unknown_decision", str(exc)) from exc
+            except ValueError as exc:
+                raise ApiError(422, "invalid_input", str(exc), "labels") from exc
+            return {"id": decision, "labels": record.labels}
+
+    @app.get("/v1/models", response_model=ModelList, dependencies=[Depends(authorize)])
+    def models() -> ModelList:
+        return ModelList(models=registry.info())
+
+    @app.get("/v1/retrain", dependencies=[Depends(authorize)])
+    def retraining() -> dict:
+        """Each served model whose adapter is a `ruling retrain` directory's `current` link, with its versions."""
+        return {"models": [{"name": name, **history(engine.adapter_path.parent)}
+                           for name, engine in registry.engines.items()
+                           if getattr(engine, "adapter_path", None) is not None and engine.adapter_path.name == CURRENT]}
+
+    @app.get("/health")
+    def health() -> dict[str, str | None]:
+        engine = registry.engines[registry.default]
+        return {"status": "ok", "model": registry.default, "revision": engine.revision}
+
+    @app.get("/", response_class=HTMLResponse, include_in_schema=False)
+    def playground(request: Request) -> str:
+        return PLAYGROUND.read_text()
+
+    @app.get("/history", response_class=HTMLResponse, include_in_schema=False)
+    def history_page() -> str:
+        return HISTORY.read_text()
+
+    return app
+
+
+def app_from_env() -> FastAPI:
+    settings = Settings.from_env()
+    retain = settings.ledger_retain_days * 86_400 if settings.ledger_retain_days is not None else None
+    ledger = DecisionLedger(settings.ledger_path, retain) if settings.ledger_path else None
+    return create_app(Registry.from_settings(settings), settings.api_key, settings.max_pending, ledger)

ruling/states.py

@@ -0,0 +1,200 @@
+"""Synthetic program state with answers computed in code.
+
+Each generator builds a JSON state, including any policy the question depends
+on, and asks Choice, Score, and Noul questions whose answers a pure function
+derives from that state. Labels are therefore exact. This is the part of the
+training mix that teaches reading structured state rather than prose.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+
+import numpy as np
+
+from ruling.evals import Record
+
+NAMES = ["Avery", "Blake", "Casey", "Devon", "Emery", "Finley", "Harper", "Jordan", "Kai", "Logan", "Morgan", "Quinn",
+         "Reese", "Rowan", "Sage", "Taylor"]
+SERVICES = ["api", "web", "auth", "billing", "search", "jobs", "gateway", "notifications", "reports", "uploads"]
+RESOURCES = ["payroll export", "customer database", "build server", "design drive", "analytics warehouse", "wiki",
+             "production logs", "finance ledger"]
+
+
+# ---- orders -------------------------------------------------------------------
+
+def orders_answers(state: dict, subject: str, threshold: float) -> dict:
+    today = dt.date.fromisoformat(state["today"])
+    orders = {o["id"]: o for o in state["orders"]}
+    order = orders[subject]
+    refundable = (order["status"] == "delivered"
+                  and (today - dt.date.fromisoformat(order["delivered_on"])).days <= state["refund_policy_days"])
+    delivered_total = sum(o["amount_usd"] for o in state["orders"] if o["status"] == "delivered")
+    return {
+        "subject_refundable": refundable,
+        "largest": max(state["orders"], key=lambda o: o["amount_usd"])["id"],
+        "pending": sum(1 for o in state["orders"] if o["status"] == "pending"),
+        "delivered_total_over": delivered_total > threshold,
+    }
+
+
+def _orders(rng: np.random.Generator) -> Record:
+    today = dt.date(2026, 1, 1) + dt.timedelta(days=int(rng.integers(0, 300)))
+    count = int(rng.integers(3, 8))
+    ids = [f"{chr(65 + int(rng.integers(26)))}{int(rng.integers(100, 999))}" for _ in range(count)]
+    ids = list(dict.fromkeys(ids))
+    orders = []
+    for oid in ids:
+        status = str(rng.choice(["pending", "shipped", "delivered", "delivered", "refunded", "cancelled"]))
+        delivered = today - dt.timedelta(days=int(rng.integers(0, 45))) if status in ("delivered", "refunded") else None
+        orders.append({"id": oid, "customer": str(rng.choice(NAMES)), "amount_usd": round(float(rng.uniform(5, 1500)), 2),
+                       "status": status, "delivered_on": delivered.isoformat() if delivered else None})
+    amounts = [o["amount_usd"] for o in orders]
+    if len(set(amounts)) != len(amounts):
+        return _orders(rng)
+    state = {"today": today.isoformat(), "refund_policy_days": int(rng.choice([7, 14, 30])), "orders": orders}
+    subject = str(rng.choice(ids))
+    threshold = float(rng.choice([500, 1000, 2000, 3000]))
+    answers = orders_answers(state, subject, threshold)
+    return Record(
+        family="state_orders",
+        state=state,
+        questions={
+            "subject_refundable": {"type": "noul", "instructions": f"Under the refund policy, order {subject} can still be refunded today."},
+            "largest": {"type": "choice", "instructions": "Which order has the largest amount?", "criteria": {i: None for i in ids}},
+            "pending": {"type": "score", "instructions": "How many orders are still pending?",
+                        "criteria": ["none", "one", "two", "three or more"]},
+            "delivered_total_over": {"type": "noul",
+                                     "instructions": f"The delivered orders add up to more than ${threshold:,.0f}."},
+        },
+        labels={**{k: answers[k] for k in ("subject_refundable", "largest", "delivered_total_over")},
+                "pending": min(answers["pending"], 3)},
+    )
+
+
+# ---- services -----------------------------------------------------------------
+
+def services_answers(state: dict, subject: str) -> dict:
+    limits = state["thresholds"]
+
+    def breach(s):
+        return s["error_rate"] > limits["error_rate"] or s["p95_ms"] > limits["p95_ms"]
+
+    services = {s["name"]: s for s in state["services"]}
+    return {
+        "subject_breach": breach(services[subject]),
+        "worst_error": max(state["services"], key=lambda s: s["error_rate"])["name"],
+        "breaches": sum(1 for s in state["services"] if breach(s)),
+    }
+
+
+def _services(rng: np.random.Generator) -> Record:
+    names = list(rng.choice(SERVICES, size=int(rng.integers(3, 7)), replace=False))
+    services = [{"name": str(n), "error_rate": round(float(rng.choice([0.0, rng.uniform(0, 0.01), rng.uniform(0.01, 0.08)])), 4),
+                 "p95_ms": int(rng.integers(40, 1600))} for n in names]
+    rates = [s["error_rate"] for s in services]
+    if len(set(rates)) != len(rates):
+        return _services(rng)
+    state = {"thresholds": {"error_rate": float(rng.choice([0.01, 0.02, 0.05])), "p95_ms": int(rng.choice([500, 800, 1200]))},
+             "services": services}
+    subject = str(rng.choice(names))
+    answers = services_answers(state, subject)
+    return Record(
+        family="state_services",
+        state=state,
+        questions={
+            "subject_breach": {"type": "noul", "instructions": f"The {subject} service is breaching at least one threshold."},
+            "worst_error": {"type": "choice", "instructions": "Which service has the highest error rate?",
+                            "criteria": {str(n): None for n in names}},
+            "breaches": {"type": "score", "instructions": "How many services breach a threshold?",
+                         "criteria": ["none", "one", "two", "three or more"]},
+        },
+        labels={"subject_breach": answers["subject_breach"], "worst_error": answers["worst_error"],
+                "breaches": min(answers["breaches"], 3)},
+    )
+
+
+# ---- access requests ----------------------------------------------------------
+
+ACCESS_POLICY = ("Internal resources: employees are permitted; contractors need manager approval. "
+                 "Restricted resources: employees need both manager and security approval; contractors are always denied.")
+
+
+def access_decision(role: str, classification: str, approvals: list[str]) -> str:
+    if classification == "internal":
+        return "permit" if role != "contractor" or "manager" in approvals else "needs_approval"
+    if role == "contractor":
+        return "deny"
+    return "permit" if {"manager", "security"} <= set(approvals) else "needs_approval"
+
+
+def _access(rng: np.random.Generator) -> Record:
+    role = str(rng.choice(["engineer", "analyst", "contractor"]))
+    classification = str(rng.choice(["internal", "restricted"]))
+    approvals = [a for a in ("manager", "security") if rng.random() < 0.5]
+    state = {"policy": ACCESS_POLICY,
+             "request": {"requester": str(rng.choice(NAMES)), "role": role, "employment": "contractor" if role == "contractor" else "employee",
+                         "resource": str(rng.choice(RESOURCES)), "classification": classification, "approvals": approvals}}
+    return Record(
+        family="state_access",
+        state=state,
+        questions={
+            "decision": {"type": "choice", "instructions": "What does the policy say to do with this access request?",
+                         "criteria": {"permit": "Grant access now", "needs_approval": "Hold until the missing approval arrives",
+                                      "deny": "Refuse the request"}},
+            "has_security": {"type": "noul", "instructions": "Security has approved this request."},
+        },
+        labels={"decision": access_decision(role, classification, approvals), "has_security": "security" in approvals},
+    )
+
+
+# ---- inventory ----------------------------------------------------------------
+
+COVER_LEVELS = ["under a week", "one to two weeks", "two to four weeks", "more than four weeks"]
+
+
+def _cover_level(days: float) -> int:
+    return 0 if days < 7 else 1 if days < 14 else 2 if days < 28 else 3
+
+
+def inventory_answers(state: dict, subject: str) -> dict:
+    items = {i["sku"]: i for i in state["items"]}
+    item = items[subject]
+    return {
+        "subject_reorder": item["on_hand"] <= item["reorder_point"],
+        "lowest_cover": min(state["items"], key=lambda i: i["on_hand"] / i["daily_sales"])["sku"],
+        "subject_cover_level": _cover_level(item["on_hand"] / item["daily_sales"]),
+    }
+
+
+def _inventory(rng: np.random.Generator) -> Record:
+    skus = list(dict.fromkeys(f"SKU-{int(rng.integers(1000, 9999))}" for _ in range(int(rng.integers(3, 7)))))
+    items = [{"sku": s, "on_hand": int(rng.integers(0, 600)), "daily_sales": int(rng.integers(1, 40)),
+              "reorder_point": int(rng.integers(20, 200))} for s in skus]
+    covers = [i["on_hand"] / i["daily_sales"] for i in items]
+    if len(set(covers)) != len(covers):
+        return _inventory(rng)
+    state = {"items": items}
+    subject = str(rng.choice(skus))
+    answers = inventory_answers(state, subject)
+    return Record(
+        family="state_inventory",
+        state=state,
+        questions={
+            "subject_reorder": {"type": "noul", "instructions": f"{subject} is at or below its reorder point."},
+            "lowest_cover": {"type": "choice", "instructions": "Which item will run out soonest at its current daily sales?",
+                             "criteria": {s: None for s in skus}},
+            "subject_cover_level": {"type": "score", "instructions": f"How long will current stock of {subject} last?",
+                                    "criteria": COVER_LEVELS},
+        },
+        labels=answers,
+    )
+
+
+GENERATORS = {"orders": _orders, "services": _services, "access": _access, "inventory": _inventory}
+
+
+def generate(count: int, seed: int) -> list[Record]:
+    rng = np.random.default_rng(seed)
+    makers = list(GENERATORS.values())
+    return [makers[i % len(makers)](rng) for i in range(count)]

ruling/thinking.py

@@ -0,0 +1,75 @@
+"""A teacher that reasons before it answers.
+
+For each question the model writes its reasoning, the reasoning is closed
+after `</think>`, and the answer is then read from the restricted letter
+logits exactly as the fast engine reads it. The result is a calibrated
+distribution informed by slow reasoning: the System Two answers a System One
+model is trained to reproduce. It is far too slow to serve.
+"""
+
+from __future__ import annotations
+
+import mlx.core as mx
+import numpy as np
+from mlx_lm import batch_generate
+from mlx_lm.sample_utils import make_sampler
+
+from ruling.calibration import Provenance, log_softmax
+from ruling.engine import Engine, InputTooLong, RawScore, Scored, answer, option_logits
+from ruling.prompt import render_question
+from ruling.questions import Choice, Noul, Score, State, SystemOneRequest, SystemOneResponse, Usage, option_keys
+
+THINK_END = "\n</think>\n\n"
+
+
+class ThinkingEngine:
+    def __init__(self, engine: Engine, max_think_tokens: int):
+        self.engine = engine
+        self.max_think_tokens = max_think_tokens
+        self.model_id = f"{engine.model_id} thinking {max_think_tokens}"
+        self.revision = engine.revision
+        self.adapter = engine.adapter
+        self.rotations = 1
+        self.prior_debias = False
+        self.max_options = engine.codes.max_options
+        self.reads_images = engine.reads_images
+        self.max_input_tokens = engine.max_input_tokens
+        self.calibration = engine.calibration
+
+    @property
+    def provenance(self) -> Provenance:
+        return Provenance(self.model_id, self.revision, self.rotations, self.prior_debias, self.adapter)
+
+    def score(self, state: State, questions: dict[str, Choice | Score | Noul]) -> Scored:
+        engine = self.engine
+        prefix, tail = engine.split_prompt(state, thinking=True)
+        prefix_tokens = engine._encode(prefix)
+        rendered = {qid: render_question(q, engine.codes.codes, 0) for qid, q in questions.items()}
+        if any(len(r.keys) > engine.codes.max_options for r in rendered.values()):
+            raise InputTooLong("a question has more options than the teacher's answer codes")
+        prompts = [prefix_tokens + engine._encode(r.text + tail) for r in rendered.values()]
+        if max(len(p) for p in prompts) + self.max_think_tokens > self.max_input_tokens:
+            raise InputTooLong(f"request needs more than {self.max_input_tokens} tokens with reasoning")
+        texts = batch_generate(engine.model, engine.tokenizer, prompts, max_tokens=self.max_think_tokens,
+                               sampler=make_sampler(temp=0.0)).texts
+        sequences = [p + engine._encode(t.split("</think>")[0].rstrip() + THINK_END) for p, t in zip(prompts, texts)]
+        width = max(len(s) for s in sequences)
+        tokens = mx.array([s + [engine._pad_id] * (width - len(s)) for s in sequences])
+        last = mx.array([len(s) - 1 for s in sequences])
+        cardinality = max(len(r.keys) for r in rendered.values())
+        ids = mx.array([engine.codes.token_ids[:cardinality] for _ in sequences])
+        logits = np.array(option_logits(engine.model, tokens, None, last, ids), dtype=np.float64)
+        raw = {}
+        for row, (qid, question) in zip(logits, questions.items()):
+            keys = rendered[qid].keys
+            log_probs = dict(zip(keys, log_softmax(row[: len(keys)])))
+            raw[qid] = RawScore(keys=option_keys(question), logits=np.array([log_probs[k] for k in option_keys(question)]))
+        return Scored(raw=raw, input_tokens=sum(len(s) for s in sequences))
+
+    def evaluate(self, request: SystemOneRequest) -> SystemOneResponse:
+        scored = self.score(request.state, request.questions)
+        answers = {qid: answer(request.questions[qid], scored.raw[qid], self.calibration) for qid in request.questions}
+        return SystemOneResponse(model=self.model_id, answers=answers, usage=Usage(input_tokens=scored.input_tokens))
+
+    def answer(self, question, raw: RawScore):
+        return answer(question, raw, self.calibration)