lazycontroller 0.1.0__py3-none-any.whl

lazycontroller/requests.py

@@ -0,0 +1,161 @@
+"""Read an OpenAI-style request into a decision state and a JSON schema, or say why it can't be.
+
+Only parameters that don't change what a correct answer is may be ignored. Anything else
+(tools, streaming, audio, logprobs, stored conversations, extra request options) means the
+caller relies on behaviour only the slow model has, so the request goes to it untouched.
+"""
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from openai import NotGiven, Omit
+
+from .schema import Ineligible
+
+# Sampling, limits and bookkeeping: they shape how a model generates, not what the answer is.
+_IGNORABLE = {
+    "frequency_penalty",
+    "max_completion_tokens",
+    "max_output_tokens",
+    "max_tokens",
+    "metadata",
+    "model",
+    "presence_penalty",
+    "prompt_cache_key",
+    "prompt_cache_retention",
+    "reasoning",
+    "reasoning_effort",
+    "safety_identifier",
+    "seed",
+    "service_tier",
+    "store",
+    "temperature",
+    "timeout",
+    "top_p",
+    "truncation",
+    "user",
+    "verbosity",
+}
+_CHAT_INPUTS = {"messages", "response_format"}
+_RESPONSES_INPUTS = {"input", "instructions", "text"}
+
+
+@dataclass(frozen=True)
+class Request:
+    state: dict[str, Any]
+    """``{"conversation": [{"role": ..., "content": ...}, ...]}``: structured, as System One
+    servers recommend. Servers that accept only a string state can't be used as the fast model."""
+    schema: Mapping[str, Any]
+    name: str | None
+
+
+def given(kwargs: Mapping[str, Any]) -> dict[str, Any]:
+    """Drop the SDK's ``omit``/``NOT_GIVEN`` placeholders."""
+    return {k: v for k, v in kwargs.items() if not isinstance(v, (Omit, NotGiven))}
+
+
+def chat_request(kwargs: Mapping[str, Any]) -> Request:
+    kwargs = given(kwargs)
+    _check_params(
+        kwargs,
+        _CHAT_INPUTS,
+        falsy={"stream", "logprobs", "parallel_tool_calls"},
+        n_ok=True,
+    )
+    response_format = kwargs.get("response_format")
+    if not isinstance(response_format, Mapping) or response_format.get("type") != "json_schema":
+        raise Ineligible("no json_schema response_format")
+    json_schema = response_format.get("json_schema") or {}
+    turns = [_chat_turn(message) for message in kwargs.get("messages") or ()]
+    return _request(turns, json_schema.get("schema"), json_schema.get("name"))
+
+
+def responses_request(kwargs: Mapping[str, Any]) -> Request:
+    kwargs = given(kwargs)
+    _check_params(kwargs, _RESPONSES_INPUTS, falsy={"stream", "background", "parallel_tool_calls"})
+    text_format = (kwargs.get("text") or {}).get("format") or {}
+    if text_format.get("type") != "json_schema":
+        raise Ineligible("no json_schema text format")
+    turns = []
+    if instructions := kwargs.get("instructions"):
+        turns.append({"role": "system", "content": _text(instructions)})
+    match kwargs.get("input"):
+        case str() as text:
+            turns.append({"role": "user", "content": text})
+        case None:
+            pass
+        case items:
+            turns.extend(_responses_turn(item) for item in items)
+    return _request(turns, text_format.get("schema"), text_format.get("name"))
+
+
+def _check_params(
+    kwargs: Mapping[str, Any], inputs: set[str], *, falsy: set[str], n_ok: bool = False
+) -> None:
+    for key, value in kwargs.items():
+        if key in inputs or key in _IGNORABLE:
+            continue
+        if key in falsy and not value:
+            continue
+        if n_ok and key == "n" and value in (None, 1):
+            continue
+        if key in {"extra_headers", "extra_query", "extra_body"} and not value:
+            continue
+        raise Ineligible(f"uses {key}")
+
+
+def _request(turns: list[dict[str, str]], schema: Any, name: str | None) -> Request:
+    if not isinstance(schema, Mapping):
+        raise Ineligible("json_schema has no schema")
+    if not any(turn["role"] == "user" for turn in turns):
+        raise Ineligible("no user message")
+    return Request({"conversation": turns}, schema, name)
+
+
+def _chat_turn(message: Any) -> dict[str, str]:
+    if not isinstance(message, Mapping):
+        raise Ineligible("message is not a dict")
+    role = message.get("role")
+    if role not in {"system", "developer", "user", "assistant"}:
+        raise Ineligible(f"{role} message")
+    if message.get("tool_calls") or message.get("function_call") or message.get("audio"):
+        raise Ineligible("assistant tool call in history")
+    return {
+        "role": "system" if role == "developer" else role,
+        "content": _text(message.get("content")),
+    }
+
+
+def _responses_turn(item: Any) -> dict[str, str]:
+    if not isinstance(item, Mapping) or item.get("type", "message") != "message":
+        raise Ineligible("non-message input item")
+    role = item.get("role")
+    if role not in {"system", "developer", "user", "assistant"}:
+        raise Ineligible(f"{role} input")
+    return {
+        "role": "system" if role == "developer" else role,
+        "content": _text(item.get("content")),
+    }
+
+
+def _text(content: Any) -> str:
+    """Flatten text content; images, audio and files can't go to the fast model as plain text."""
+    match content:
+        case str():
+            return content
+        case None:
+            return ""
+        case Iterable():
+            parts = []
+            for part in content:
+                if not isinstance(part, Mapping) or part.get("type") not in {
+                    "text",
+                    "input_text",
+                    "output_text",
+                }:
+                    raise Ineligible("non-text content part")
+                parts.append(part.get("text") or "")
+            return "\n".join(parts)
+        case _:
+            raise Ineligible("unrecognized message content")

lazycontroller/schema.py

@@ -0,0 +1,298 @@
+"""Turn a structured-output JSON schema into decision questions, and the answers back into JSON.
+
+A schema is eligible only when every value it asks for is a closed decision:
+
+- ``enum`` (from ``Literal``, ``Enum``, ...) becomes a Choice;
+- ``boolean`` becomes a Noul;
+- ``integer`` with a small ``minimum``..``maximum`` range becomes a Score, one level per value;
+- ``const`` is filled in without asking;
+- objects are walked field by field, and ``X | None`` of an enum or boolean becomes a Choice
+  with a ``null`` option.
+
+Anything else (free strings, arrays, unbounded numbers, unions) needs a generative model, so the
+whole request is left to the slow model.
+"""
+
+import json
+import re
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from typesafe_sdk import (
+    Answer,
+    Choice,
+    ChoiceAnswer,
+    Noul,
+    NoulAnswer,
+    Score,
+    ScoreAnswer,
+)
+
+MAX_SCORE_LEVELS = 11
+"""Integer ranges wider than this are treated as open numbers, not a rubric."""
+
+_MAX_DEPTH = 16
+
+# Keywords whose meaning we honour. Any other constraint (multipleOf, pattern, format, ...)
+# could reject a value the fast model picks, so it makes the request ineligible rather than ignored.
+_ANNOTATIONS = {"title", "description", "default", "examples", "$comment", "deprecated"}
+_KEYWORDS = _ANNOTATIONS | {
+    "type",
+    "enum",
+    "const",
+    "properties",
+    "required",
+    "additionalProperties",
+    "minimum",
+    "maximum",
+    "exclusiveMinimum",
+    "exclusiveMaximum",
+    "$defs",
+    "definitions",
+}
+
+type Path = tuple[str, ...]
+type Decoder = Callable[[Any], tuple[Any, float]]
+
+
+class Ineligible(Exception):
+    """The fast model can't answer this request; the message says why."""
+
+
+@dataclass(frozen=True)
+class Field:
+    path: Path
+    question: Choice | Noul | Score
+    decode: Decoder
+    """Maps the decision answer to (JSON value, confidence in 0..1)."""
+
+
+@dataclass(frozen=True)
+class Plan:
+    """The decision questions for one schema, plus the values that need no question."""
+
+    fields: tuple[Field, ...]
+    constants: tuple[tuple[Path, Any], ...]
+
+    @property
+    def questions(self) -> dict[str, Choice | Noul | Score]:
+        return {_question_id(i, f.path): f.question for i, f in enumerate(self.fields)}
+
+    def assemble(self, answers: Mapping[str, Answer]) -> tuple[dict[str, Any], dict[str, float]]:
+        """Build the output object; also return each field's confidence keyed by dotted path."""
+        output: dict[str, Any] = {}
+        confidences: dict[str, float] = {}
+        for path, value in self.constants:
+            _put(output, path, value)
+        for i, f in enumerate(self.fields):
+            value, confidence = f.decode(answers[_question_id(i, f.path)])
+            _put(output, f.path, value)
+            confidences[".".join(f.path)] = confidence
+        return output, confidences
+
+
+def compile_schema(schema: Mapping[str, Any], *, title: str | None = None) -> Plan:
+    """Raise Ineligible unless every value in ``schema`` maps to a decision question."""
+    fields: list[Field] = []
+    constants: list[tuple[Path, Any]] = []
+    _walk(schema, schema, (), title or schema.get("title"), fields, constants, 0)
+    if not fields:
+        raise Ineligible("schema has no fields to decide")
+    return Plan(tuple(fields), tuple(constants))
+
+
+def _walk(
+    node: Mapping[str, Any],
+    root: Mapping[str, Any],
+    path: Path,
+    title: str | None,
+    fields: list[Field],
+    constants: list[tuple[Path, Any]],
+    depth: int,
+) -> None:
+    if depth > _MAX_DEPTH:
+        raise Ineligible("schema is nested too deeply")
+    node = _resolve(node, root)
+    where = ".".join(path) or "<root>"
+
+    if "const" in node:
+        if not path:
+            raise Ineligible("root schema is a constant")
+        constants.append((path, node["const"]))
+        return
+
+    if _type(node) == "object" or "properties" in node:
+        if not path and _type(node) != "object":
+            raise Ineligible("root schema is not an object")
+        properties = node.get("properties") or {}
+        if not properties:
+            raise Ineligible(f"{where}: object without properties")
+        if node.get("patternProperties") or isinstance(node.get("additionalProperties"), dict):
+            raise Ineligible(f"{where}: open-ended object")
+        _check_keywords(node, where)
+        for name, child in properties.items():
+            _walk(child, root, (*path, name), title, fields, constants, depth + 1)
+        return
+
+    if not path:
+        raise Ineligible("root schema is not an object")
+    fields.append(_leaf(node, root, path, title))
+
+
+def _leaf(node: Mapping[str, Any], root: Mapping[str, Any], path: Path, title: str | None) -> Field:
+    where = ".".join(path)
+    instructions = _instructions(node, path, title)
+
+    nullable, node = _split_nullable(node, root)
+    field = _leaf_field(node, path, instructions, nullable)
+    _check_keywords(node, where)
+    return field
+
+
+def _leaf_field(node: Mapping[str, Any], path: Path, instructions: str, nullable: bool) -> Field:
+    where = ".".join(path)
+    if "enum" in node:
+        return _choice(path, instructions, list(node["enum"]), nullable)
+    match _type(node):
+        case "boolean" if nullable:
+            return _choice(path, instructions, [True, False], nullable=True)
+        case "boolean":
+            return Field(path, Noul(instructions=instructions), _decode_noul)
+        case "integer" if not nullable:
+            return _score(path, instructions, node)
+        case "string":
+            raise Ineligible(f"{where}: free-text string")
+        case "array":
+            raise Ineligible(f"{where}: array")
+        case "number" | "integer":
+            raise Ineligible(f"{where}: open-ended number")
+        case _:
+            raise Ineligible(f"{where}: unsupported schema {json.dumps(node)[:80]}")
+
+
+def _choice(path: Path, instructions: str, values: list[Any], nullable: bool) -> Field:
+    if nullable and None not in values:
+        values = [*values, None]
+    if not values:
+        raise Ineligible(f"{'.'.join(path)}: empty enum")
+    labels = {json.dumps(v) if not isinstance(v, str) else v: v for v in values}
+    if len(labels) != len(values):
+        raise Ineligible(f"{'.'.join(path)}: enum values collide as labels")
+    criteria = {
+        label: ("Not stated or not applicable" if v is None else None)
+        for label, v in labels.items()
+    }
+
+    def decode(answer: Any) -> tuple[Any, float]:
+        if not isinstance(answer, ChoiceAnswer) or answer.choice not in labels:
+            raise TypeError(f"expected a choice among {list(labels)}, got {answer!r}")
+        return labels[answer.choice], answer.confidence
+
+    return Field(path, Choice(instructions=instructions, criteria=criteria), decode)
+
+
+def _decode_noul(answer: Any) -> tuple[Any, float]:
+    if not isinstance(answer, NoulAnswer):
+        raise TypeError(f"expected a noul, got {answer!r}")
+    # Nouls carry no confidence; |2p - 1| is the one TypeSafe suggests, on the Choice scale.
+    return answer.noul >= 0.5, abs(2 * answer.noul - 1)
+
+
+def _score(path: Path, instructions: str, node: Mapping[str, Any]) -> Field:
+    where = ".".join(path)
+    low, high = node.get("minimum"), node.get("maximum")
+    if "exclusiveMinimum" in node:
+        low = node["exclusiveMinimum"] + 1
+    if "exclusiveMaximum" in node:
+        high = node["exclusiveMaximum"] - 1
+    if not isinstance(low, int) or not isinstance(high, int):
+        raise Ineligible(f"{where}: integer without minimum and maximum")
+    if not 2 <= high - low + 1 <= MAX_SCORE_LEVELS:
+        raise Ineligible(f"{where}: integer range {low}..{high} is not a small rubric")
+    levels = [f"{path[-1]} = {value}" for value in range(low, high + 1)]
+
+    def decode(answer: Any) -> tuple[Any, float]:
+        if not isinstance(answer, ScoreAnswer):
+            raise TypeError(f"expected a score, got {answer!r}")
+        # The most likely level, not the expected score: the schema wants one of its integers.
+        level = max(answer.probabilities, key=answer.probabilities.__getitem__)
+        return low + level, answer.confidence
+
+    return Field(path, Score(instructions=instructions, criteria=levels), decode)
+
+
+def _split_nullable(
+    node: Mapping[str, Any], root: Mapping[str, Any]
+) -> tuple[bool, Mapping[str, Any]]:
+    """Unwrap ``X | None`` written as anyOf/oneOf or as a ``type`` list."""
+    for key in ("anyOf", "oneOf"):
+        if key in node:
+            options = [_resolve(o, root) for o in node[key]]
+            rest = [o for o in options if _type(o) != "null"]
+            if len(rest) != 1:
+                raise Ineligible("union of several types")
+            merged = {**{k: v for k, v in node.items() if k != key}, **rest[0]}
+            return len(rest) < len(options), merged
+    types = node.get("type")
+    if isinstance(types, list):
+        rest = [t for t in types if t != "null"]
+        if len(rest) != 1:
+            raise Ineligible("union of several types")
+        return len(rest) < len(types), {**node, "type": rest[0]}
+    return False, node
+
+
+def _check_keywords(node: Mapping[str, Any], where: str) -> None:
+    unknown = sorted(set(node) - _KEYWORDS)
+    if unknown:
+        raise Ineligible(f"{where}: unsupported keyword {', '.join(unknown)}")
+    if node.get("additionalProperties") not in (None, False, True):
+        raise Ineligible(f"{where}: open-ended object")
+
+
+def _resolve(node: Mapping[str, Any], root: Mapping[str, Any]) -> Mapping[str, Any]:
+    seen = 0
+    while "$ref" in node:
+        ref = node["$ref"]
+        if not ref.startswith("#/"):
+            raise Ineligible(f"external $ref {ref}")
+        target: Any = root
+        for part in ref[2:].split("/"):
+            target = target[part.replace("~1", "/").replace("~0", "~")]
+        # Sibling keywords (e.g. a field description next to an Enum $ref) win over the target.
+        node = {**target, **{k: v for k, v in node.items() if k != "$ref"}}
+        seen += 1
+        if seen > _MAX_DEPTH:
+            raise Ineligible("recursive $ref")
+    if "allOf" in node and len(node["allOf"]) == 1:
+        inner = _resolve(node["allOf"][0], root)
+        node = {**inner, **{k: v for k, v in node.items() if k != "allOf"}}
+    if "allOf" in node:
+        raise Ineligible("allOf with several schemas")
+    return node
+
+
+def _type(node: Mapping[str, Any]) -> Any:
+    return node.get("type")
+
+
+def _instructions(node: Mapping[str, Any], path: Path, title: str | None) -> str:
+    output = title or "structured output"
+    subject = f"the `{'.'.join(path)}` field of the {output} the conversation asks for"
+    description = node.get("description") or _humanize(path[-1])
+    return f"Decide {subject}: {description}"
+
+
+def _humanize(name: str) -> str:
+    return re.sub(r"(?<=[a-z0-9])(?=[A-Z])|_+", " ", name).strip().lower()
+
+
+def _question_id(index: int, path: Path) -> str:
+    return f"f{index}_" + re.sub(r"[^A-Za-z0-9_]", "_", "__".join(path))[:60]
+
+
+def _put(output: dict[str, Any], path: Path, value: Any) -> None:
+    for key in path[:-1]:
+        output = output.setdefault(key, {})
+    output[path[-1]] = value

lazycontroller/shadow.py

@@ -0,0 +1,126 @@
+"""Turn shadow-mode routes into the numbers you need to pick ``min_confidence``.
+
+In shadow mode both models answer every call; each route records the fast model's confidence
+and whether the two agreed. ``ShadowReport`` answers "if ``min_confidence`` were t, how much
+traffic would the fast model take, and how often would it agree with the slow one?", and
+suggests the lowest value that keeps agreement where you want it.
+"""
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+from .lazy import Route
+
+
+@dataclass(frozen=True)
+class Point:
+    """What the fast model would take at one ``min_confidence``."""
+
+    min_confidence: float
+    taken: int
+    """Calls whose confidence reaches ``min_confidence``."""
+    agreed: int
+    """Of those, the ones where the fast and slow answers matched."""
+    total: int
+    """All shadow calls in the report."""
+
+    @property
+    def coverage(self) -> float:
+        """Share of all calls the fast model would answer."""
+        return self.taken / self.total if self.total else 0.0
+
+    @property
+    def agreement(self) -> float | None:
+        """Share of the taken calls where the two models agreed; None if none are taken."""
+        return self.agreed / self.taken if self.taken else None
+
+
+@dataclass(frozen=True)
+class Band:
+    """Calls whose confidence falls in ``[low, high)``."""
+
+    low: float
+    high: float
+    count: int
+    agreed: int
+
+    @property
+    def agreement(self) -> float | None:
+        return self.agreed / self.count if self.count else None
+
+
+_GRID = tuple(round(i * 0.05, 2) for i in range(21))  # 0.00, 0.05, ..., 1.00
+
+
+class ShadowReport:
+    """Agreement between the fast and slow models, by confidence, from shadow-mode routes.
+
+    Routes that aren't from shadow mode (or have no confidence) are ignored, so the list
+    collected by ``on_route`` can be passed as is.
+    """
+
+    def __init__(self, routes: Iterable[Route]) -> None:
+        self._calls = tuple(
+            (r.confidence, not r.shadow_mismatches)
+            for r in routes
+            if r.reason == "shadow" and r.confidence is not None
+        )
+
+    def __len__(self) -> int:
+        return len(self._calls)
+
+    def at(self, min_confidence: float) -> Point:
+        taken = [agreed for confidence, agreed in self._calls if confidence >= min_confidence]
+        return Point(min_confidence, len(taken), sum(taken), len(self._calls))
+
+    def curve(self, values: Sequence[float] = _GRID) -> list[Point]:
+        """One point per ``min_confidence`` value, by default every 0.05 from 0 to 1. Ready to
+        plot."""
+        return [self.at(v) for v in values]
+
+    def bands(self, edges: Sequence[float] = (0.0, 0.7, 0.9, 1.0)) -> list[Band]:
+        """Calls grouped by confidence, highest band first; the top band includes 1.0."""
+        bands = []
+        for low, high in zip(edges, edges[1:], strict=False):
+            top = high >= edges[-1]
+            inside = [a for c, a in self._calls if low <= c and (c <= high if top else c < high)]
+            bands.append(Band(low, high, len(inside), sum(inside)))
+        return bands[::-1]
+
+    def suggest_min_confidence(
+        self, min_agreement: float, *, min_samples: int = 20
+    ) -> float | None:
+        """The lowest ``min_confidence`` at which agreement stays at or above ``min_agreement``.
+
+        "Stays" means at that value and at every higher one that still covers at least
+        ``min_samples`` calls, so the answer doesn't rest on a lucky dip. Candidates are the
+        observed confidences. Returns None when no value covers ``min_samples`` calls with
+        enough agreement: collect more shadow traffic, or lower your bar.
+        """
+        if not 0.0 <= min_agreement <= 1.0:
+            raise ValueError("min_agreement must be between 0 and 1")
+        points = [self.at(c) for c in sorted({c for c, _ in self._calls})]
+        points = [p for p in points if p.taken >= min_samples]
+        best = None
+        for point in reversed(points):  # from the highest value down
+            if point.agreement < min_agreement:
+                break
+            best = point.min_confidence
+        return best
+
+    def table(self, values: Sequence[float] = (0.5, 0.6, 0.7, 0.8, 0.9)) -> str:
+        """A plain-text summary: agreement by confidence band, then by ``min_confidence``."""
+        lines = [f"{len(self)} shadow calls", "", "confidence band   calls  agreement"]
+        for band in self.bands():
+            agreement = _ratio(band.agreed, band.count)
+            lines.append(f"  {band.low:.2f}-{band.high:.2f}       {band.count:>5}  {agreement}")
+        lines += ["", "min confidence   fast takes      agreement"]
+        for p in self.curve(values):
+            share = f"{p.taken}/{p.total} ({p.coverage:.0%})"
+            agreement = _ratio(p.agreed, p.taken)
+            lines.append(f"  {p.min_confidence:.2f}           {share:<14}  {agreement}")
+        return "\n".join(lines)
+
+
+def _ratio(part: int, whole: int) -> str:
+    return f"{part}/{whole} ({part / whole:.0%})" if whole else "-"