sys1bench 0.3.0__py3-none-any.whl

sys1bench/__init__.py

@@ -0,0 +1,3 @@
+"""sys1bench: benchmark harness for typed System One decision models."""
+
+__version__ = "0.3.0"

sys1bench/adapters/__init__.py

@@ -0,0 +1,19 @@
+"""Model adapters. Register new models with `@register("id")` or via the
+`sys1bench.adapters` entry-point group so future models need no changes here."""
+
+from . import (  # noqa: F401
+    generic_http,
+    hybrid_router,
+    jev_openrouter,
+    jev_typesafe,
+    laya_local,
+    majority_prior,
+    mock,
+    regex_keyword,
+)
+from .base import BaseAdapter, get_adapter, list_adapters, register  # noqa: F401
+
+try:  # optional heavy deps
+    from . import embed_knn, encoder_finetuned, llm_constrained, nli_zeroshot  # noqa: F401
+except Exception:  # pragma: no cover
+    pass

sys1bench/adapters/base.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import importlib.metadata as md
+import time
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any
+
+import numpy as np
+
+from ..schemas import Answer, DecisionRequest, DecisionResponse, ModelCapabilities, Question
+
+_REGISTRY: dict[str, type[BaseAdapter]] = {}
+
+
+class FatalAdapterError(RuntimeError):
+    """Raised when an adapter cannot produce valid measurements at all (e.g. a local model refused to load on the
+    requested device). The runner does not swallow it: the run aborts so an orchestrator can retry or fail loudly."""
+
+
+def register(adapter_id: str) -> Callable[[type[BaseAdapter]], type[BaseAdapter]]:
+    def deco(cls: type[BaseAdapter]) -> type[BaseAdapter]:
+        cls.adapter_id = adapter_id
+        _REGISTRY[adapter_id] = cls
+        return cls
+
+    return deco
+
+
+def _load_entry_points() -> None:
+    try:
+        eps = md.entry_points(group="sys1bench.adapters")
+    except Exception:  # pragma: no cover
+        return
+    for ep in eps:
+        if ep.name not in _REGISTRY:
+            try:
+                cls = ep.load()
+                cls.adapter_id = ep.name
+                _REGISTRY[ep.name] = cls
+            except Exception:  # pragma: no cover
+                continue
+
+
+def list_adapters() -> list[str]:
+    _load_entry_points()
+    return sorted(_REGISTRY)
+
+
+def get_adapter(adapter_id: str, **config: Any) -> BaseAdapter:
+    _load_entry_points()
+    if adapter_id not in _REGISTRY:
+        raise KeyError(f"unknown adapter {adapter_id!r}; known: {list_adapters()}")
+    return _REGISTRY[adapter_id](**config)
+
+
+def detect_quantisation(probs: list[float]) -> float | None:
+    """Smallest decimal step that reproduces every probability (0.01, 0.001, ...), or None if finer than 1e-6."""
+    arr = np.asarray(probs, dtype=float)
+    if not np.all(np.isfinite(arr)):
+        return None
+    for step in (0.1, 0.05, 0.01, 0.005, 0.001, 1e-4, 1e-5, 1e-6):
+        if np.allclose(np.round(arr / step) * step, arr, atol=step * 1e-3):
+            return step
+    return None
+
+
+SUM_EXACT_TOL = 1e-4   # sums within this are taken as-is
+SUM_SLACK_TOL = 0.02   # sums within this are rescaled and flagged `renormalised` (vendors quantise to 0.01)
+
+
+def finalize_answer(q: Question, probs: list[float], *, confidence: float | None = None,
+                    abstained: bool = False, truncated: bool = False, tol: float = SUM_SLACK_TOL) -> Answer:
+    """Validate a probability vector against the contract. Vectors whose sum is off by more than `tol` are recorded as
+    schema failures. Vectors off by less (quantisation slack, e.g. 0.99 from 0.01-rounded probabilities) are rescaled
+    and flagged `renormalised=True` with the raw sum kept, so the rate is reportable and nothing is hidden."""
+    keys = q.option_keys
+    if len(probs) != len(keys):
+        return Answer.failed(q, f"probs length {len(probs)} != options {len(keys)}")
+    arr = np.asarray(probs, dtype=float)
+    if not np.all(np.isfinite(arr)):
+        return Answer.failed(q, "non-finite probability")
+    if np.any(arr < -tol) or np.any(arr > 1 + tol):
+        return Answer.failed(q, "probability outside [0,1]")
+    s = float(arr.sum())
+    if abs(s - 1.0) > tol or s <= 0:
+        return Answer.failed(q, f"probabilities sum to {s:.6f}")
+    step = detect_quantisation(list(arr))
+    renorm = abs(s - 1.0) > SUM_EXACT_TOL
+    if renorm:
+        arr = arr / s
+    arr = np.clip(arr, 0.0, 1.0)
+    return Answer(
+        type=q.type,
+        option_keys=keys,
+        probs=[float(x) for x in arr],
+        argmax=keys[int(arr.argmax())],
+        confidence=confidence,
+        abstained=abstained,
+        truncated=truncated,
+        renormalised=renorm,
+        raw_prob_sum=s,
+        quantisation_step=step,
+    )
+
+
+class BaseAdapter(ABC):
+    adapter_id: str = "base"
+
+    def __init__(self, model_id: str = "", **tunables: Any) -> None:
+        self.model_id = model_id
+        self.tunables: dict[str, Any] = tunables
+
+    @property
+    @abstractmethod
+    def capabilities(self) -> ModelCapabilities: ...
+
+    @abstractmethod
+    def decide(self, request: DecisionRequest) -> DecisionResponse: ...
+
+    def decide_batch(self, requests: list[DecisionRequest]) -> list[DecisionResponse]:
+        return [self.decide(r) for r in requests]
+
+    def reparse(self, request: DecisionRequest, cached: DecisionResponse) -> DecisionResponse | None:
+        """Rebuild answers from `cached.raw` with the current parser (no API call). Adapters that store the vendor
+        payload verbatim override `parse_raw_answer`; return None if nothing can be rebuilt."""
+        raw_answers = (cached.raw or {}).get("answers") if isinstance(cached.raw, dict) else None
+        if not isinstance(raw_answers, dict):
+            return None
+        answers = {}
+        for k, q in request.questions.items():
+            payload = raw_answers.get(k)
+            answers[k] = self.parse_raw_answer(q, payload) if isinstance(payload, dict) else Answer.failed(q, "missing answer")
+        return cached.model_copy(update={"answers": answers})
+
+    def parse_raw_answer(self, q: Question, payload: dict) -> Answer:  # pragma: no cover - overridden
+        raise NotImplementedError
+
+    def warmup(self) -> None:  # pragma: no cover
+        return None
+
+    def check_request(self, request: DecisionRequest) -> list[str]:
+        """Capability violations for this request (reported, not fatal)."""
+        caps = self.capabilities
+        issues: list[str] = []
+        if caps.max_questions_per_request and len(request.questions) > caps.max_questions_per_request:
+            issues.append("too_many_questions")
+        for k, q in request.questions.items():
+            if q.type not in caps.primitives:
+                issues.append(f"{k}:unsupported_primitive")
+            if q.type == "choice" and caps.max_options and q.cardinality > caps.max_options:
+                issues.append(f"{k}:too_many_options")
+            if q.type == "score" and caps.max_score_levels and q.cardinality > caps.max_score_levels:
+                issues.append(f"{k}:too_many_levels")
+        return issues
+
+    @staticmethod
+    def _now_ms() -> float:
+        return time.perf_counter() * 1000.0

sys1bench/adapters/embed_knn.py

@@ -0,0 +1,61 @@
+"""Embedding baseline: embed the state and each option description, softmax over cosine similarity
+with a temperature fitted on a calibration split. This is the honest apples-to-apples for
+"zero-shot from label descriptions"; a System One model should beat it by a clear margin."""
+
+from __future__ import annotations
+
+import time
+
+import numpy as np
+
+from ..schemas import DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("embed_knn")
+class EmbedKNNAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "BAAI/bge-m3", temperature: float = 0.05, device: str | None = None, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.temperature, self.device = temperature, device
+        self._m = None
+        self._cache: dict[str, np.ndarray] = {}
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=f"embed_knn:{self.model_id}", deployment="local", supports_batching=True,
+                                 tunables={"temperature": self.temperature})
+
+    def _model(self):
+        if self._m is None:
+            from sentence_transformers import SentenceTransformer  # type: ignore
+
+            self._m = SentenceTransformer(self.model_id, device=self.device)
+        return self._m
+
+    def _embed(self, texts: list[str]) -> np.ndarray:
+        todo = [t for t in texts if t not in self._cache]
+        if todo:
+            vecs = self._model().encode(todo, normalize_embeddings=True, convert_to_numpy=True)
+            for t, v in zip(todo, vecs):
+                self._cache[t] = v
+        return np.stack([self._cache[t] for t in texts])
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = time.perf_counter()
+        s = self._embed([request.state_text()])[0]
+        answers = {}
+        for k, q in request.questions.items():
+            if q.type == "noul":
+                texts = [f"{q.instructions} Yes.", f"{q.instructions} No."]
+            elif q.type == "choice":
+                texts = [f"{c.key}: {c.description}" if c.description else c.key for c in q.criteria]  # type: ignore[union-attr]
+            else:
+                texts = [f"level {c.level}: {c.description}" for c in q.criteria]  # type: ignore[union-attr]
+            sims = self._embed(texts) @ s
+            z = sims / self.temperature
+            p = np.exp(z - z.max())
+            p /= p.sum()
+            answers[k] = finalize_answer(q, [float(x) for x in p], confidence=float(p.max()))
+        ms = (time.perf_counter() - t0) * 1000
+        return DecisionResponse(answers=answers, latency=LatencyRecord(client_ms=ms, compute_ms=ms, timestamp=time.time()),
+                                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, cost_usd=0.0))

sys1bench/adapters/encoder_finetuned.py

@@ -0,0 +1,114 @@
+"""Fine-tuned encoder baseline: a small classifier (DistilBERT by default) trained per question on a *disjoint* seed of
+the same generator, then evaluated on the benchmark manifest. This is the "strongest cheap baseline" from jevbench:
+it has seen thousands of labelled examples where the System One models see only descriptions, and the report says so.
+
+    adapter: encoder_finetuned
+    model_id: distilbert-base-uncased
+    train_generator: support_tickets
+    train_n: 4000
+    train_seed: 7                      # never the benchmark seed
+    epochs: 2
+Trains one head per question key at first use (cached under ~/.cache/sys1bench/encoders/<hash>).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+
+from ..schemas import Answer, DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("encoder_finetuned")
+class EncoderFinetunedAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "distilbert-base-uncased", train_generator: str = "support_tickets", train_n: int = 4000,
+                 train_seed: int = 7, epochs: int = 2, batch_size: int = 32, lr: float = 5e-5, max_length: int = 256,
+                 device: str | None = None, cache_dir: str | None = None, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.train_generator, self.train_n, self.train_seed, self.epochs = train_generator, train_n, train_seed, epochs
+        self.batch_size, self.lr, self.max_length, self.device = batch_size, lr, max_length, device
+        self.cache_dir = Path(cache_dir or Path.home() / ".cache" / "sys1bench" / "encoders")
+        self.tunables.update({"train_generator": train_generator, "train_n": train_n, "train_seed": train_seed, "epochs": epochs})
+        self._models: dict[str, tuple] = {}
+        self._tok = None
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=f"encoder_ft:{self.model_id}:{self.train_generator}", deployment="local", supports_batching=True,
+                                 tunables={"train_n": self.train_n, "train_seed": self.train_seed, "note": "trained on labelled examples; not zero-shot"})
+
+    def _key(self, qkey: str, labels: list[str]) -> str:
+        return hashlib.sha256(json.dumps([self.model_id, self.train_generator, self.train_n, self.train_seed, self.epochs, qkey, labels]).encode()).hexdigest()[:16]
+
+    def _ensure(self, qkey: str, labels: list[str], qtype: str):
+        if qkey in self._models:
+            return self._models[qkey]
+        import torch  # type: ignore
+        from transformers import AutoModelForSequenceClassification, AutoTokenizer  # type: ignore
+
+        from ..generators import get_generator
+
+        dev = self.device or ("cuda" if torch.cuda.is_available() else "cpu")
+        if self._tok is None:
+            self._tok = AutoTokenizer.from_pretrained(self.model_id)
+        path = self.cache_dir / self._key(qkey, labels)
+        if path.exists():
+            model = AutoModelForSequenceClassification.from_pretrained(path).to(dev).eval()
+        else:
+            items = get_generator(self.train_generator, n=self.train_n, seed=self.train_seed).generate()
+            texts, ys = [], []
+            for it in items:
+                q = it.questions.get(qkey)
+                if q is None or q.ground_truth is None:
+                    continue
+                gt = str(q.ground_truth).lower() if q.type == "noul" else str(q.ground_truth)
+                if gt in labels:
+                    texts.append(it.state_text())
+                    ys.append(labels.index(gt))
+            model = AutoModelForSequenceClassification.from_pretrained(self.model_id, num_labels=len(labels)).to(dev)
+            opt = torch.optim.AdamW(model.parameters(), lr=self.lr)
+            model.train()
+            idx = np.arange(len(texts))
+            rng = np.random.default_rng(self.train_seed)
+            for _ in range(self.epochs):
+                rng.shuffle(idx)
+                for b in range(0, len(idx), self.batch_size):
+                    bi = idx[b:b + self.batch_size]
+                    enc = self._tok([texts[i] for i in bi], truncation=True, max_length=self.max_length, padding=True, return_tensors="pt").to(dev)
+                    out = model(**enc, labels=torch.tensor([ys[i] for i in bi], device=dev))
+                    out.loss.backward()
+                    opt.step()
+                    opt.zero_grad()
+            model.eval()
+            path.mkdir(parents=True, exist_ok=True)
+            model.save_pretrained(path)
+        self._models[qkey] = (model, dev)
+        return self._models[qkey]
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        import torch  # type: ignore
+
+        t0 = self._now_ms()
+        answers = {}
+        hw = None
+        for k, q in request.questions.items():
+            labels = q.option_keys
+            try:
+                model, dev = self._ensure(k, labels, q.type)
+            except Exception as e:
+                answers[k] = Answer.failed(q, f"train_failed:{type(e).__name__}")
+                continue
+            enc = self._tok([request.state_text()], truncation=True, max_length=self.max_length, padding=True, return_tensors="pt").to(dev)
+            with torch.no_grad():
+                p = torch.softmax(model(**enc).logits[0].float(), -1).cpu().numpy()
+            answers[k] = finalize_answer(q, [float(x) for x in p], confidence=float(p.max()))
+            hw = hw or (torch.cuda.get_device_name(0) if "cuda" in str(dev) else "cpu")
+        ms = self._now_ms() - t0
+        return DecisionResponse(answers=answers, latency=LatencyRecord(client_ms=ms, compute_ms=ms, timestamp=time.time()),
+                                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, model_id_returned=f"encoder_ft:{self.model_id}",
+                                                        version_hash=f"{self.train_generator}@n{self.train_n}s{self.train_seed}e{self.epochs}", hardware=hw, cost_usd=0.0))

sys1bench/adapters/generic_http.py

@@ -0,0 +1,161 @@
+"""Config-driven adapter for future hosted System One models.
+
+A new vendor is added with a YAML file, not code:
+
+    adapter: generic_http
+    model_id: vendor/model-1.0
+    url: https://api.vendor.com/v1/decide
+    auth_env: VENDOR_API_KEY
+    type_map: {choice: choice, score: score, noul: boolean}
+    body_template: {"model": "{model_id}", "input": "{state}", "questions": "{questions}"}
+    answers_path: ["result", "answers"]
+    fields: {probs: ["probabilities", "distribution"], p_true: ["probability"], confidence: ["confidence"],
+             model_returned: ["model"], generation_id: ["id"], input_tokens: ["usage", "input_tokens"]}
+    price_per_m_input: 0.05
+
+Anything the template cannot express is a real adapter; this covers the common JSON shape.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import time
+from typing import Any
+
+import httpx
+
+from ..schemas import (
+    Answer,
+    DecisionRequest,
+    DecisionResponse,
+    LatencyRecord,
+    ModelCapabilities,
+    ProviderRecord,
+    Question,
+)
+from .base import BaseAdapter, finalize_answer, register
+
+
+def _dig(d: Any, path: list[str] | str | None, default=None):
+    if path is None:
+        return default
+    if isinstance(path, str):
+        path = [path]
+    cur = d
+    for p in path:
+        if not isinstance(cur, dict) or p not in cur:
+            return default
+        cur = cur[p]
+    return cur
+
+
+def _first_path(d: Any, paths: list[Any] | None, default=None):
+    for p in paths or []:
+        v = _dig(d, p)
+        if v is not None:
+            return v
+    return default
+
+
+@register("generic_http")
+class GenericHTTPAdapter(BaseAdapter):
+    def __init__(self, model_id: str, url: str, auth_env: str | None = None, headers: dict | None = None,
+                 type_map: dict | None = None, body_template: dict | None = None, answers_path: list | None = None,
+                 fields: dict | None = None, price_per_m_input: float | None = None, deployment: str = "hosted",
+                 max_options: int | None = None, max_state_tokens: int | None = None, timeout_s: float = 30.0,
+                 max_retries: int = 5, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.url, self.timeout_s, self.max_retries = url, timeout_s, max_retries
+        self.headers = dict(headers or {})
+        if auth_env and os.environ.get(auth_env):
+            self.headers.setdefault("Authorization", f"Bearer {os.environ[auth_env]}")
+        self.type_map = type_map or {"choice": "choice", "score": "score", "noul": "boolean"}
+        self.body_template = body_template or {"model": "{model_id}", "state": "{state}", "questions": "{questions}"}
+        self.answers_path = answers_path or ["answers"]
+        self.fields = fields or {}
+        self.price = price_per_m_input
+        self._caps = ModelCapabilities(name=model_id, deployment=deployment, max_options=max_options,  # type: ignore[arg-type]
+                                       max_state_tokens=max_state_tokens, supports_batching=True)
+        self._client = httpx.Client(timeout=timeout_s)
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return self._caps
+
+    def _vendor_q(self, q: Question) -> dict[str, Any]:
+        out: dict[str, Any] = {"type": self.type_map[q.type], "instructions": q.instructions}
+        if q.type == "choice":
+            out["criteria"] = [{"key": c.key, "description": c.description} for c in q.criteria]  # type: ignore[union-attr]
+        elif q.type == "score":
+            out["criteria"] = [{"level": c.level, "description": c.description} for c in q.criteria]  # type: ignore[union-attr]
+        return out
+
+    def _render(self, request: DecisionRequest) -> dict[str, Any]:
+        subs = {"{model_id}": self.model_id, "{state}": request.state,
+                "{questions}": {k: self._vendor_q(q) for k, q in request.questions.items()}}
+
+        def walk(x):
+            if isinstance(x, str) and x in subs:
+                return subs[x]
+            if isinstance(x, str):
+                return x.replace("{model_id}", self.model_id)
+            if isinstance(x, dict):
+                return {k: walk(v) for k, v in x.items()}
+            if isinstance(x, list):
+                return [walk(v) for v in x]
+            return x
+
+        return walk(json.loads(json.dumps(self.body_template)))
+
+    def _parse(self, q: Question, payload: dict[str, Any]) -> Answer:
+        keys = q.option_keys
+        conf = _first_path(payload, self.fields.get("confidence", [["confidence"]]))
+        if q.type == "noul":
+            p = _first_path(payload, self.fields.get("p_true", [["probability"], ["p_true"]]))
+            return Answer.failed(q, "no probability") if p is None else finalize_answer(q, [float(p), 1 - float(p)], confidence=conf)
+        dist = _first_path(payload, self.fields.get("probs", [["probabilities"], ["distribution"]]))
+        if isinstance(dist, dict):
+            probs = [float(dist.get(k, 0.0)) for k in keys]
+        elif isinstance(dist, list):
+            probs = [float(x) for x in dist[: len(keys)]]
+        else:
+            return Answer.failed(q, "no distribution")
+        return finalize_answer(q, probs, confidence=conf)
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        body = self._render(request)
+        delay, last = 0.5, None
+        for _ in range(self.max_retries):
+            t0 = time.perf_counter()
+            try:
+                r = self._client.post(self.url, json=body, headers=self.headers)
+                ms = (time.perf_counter() - t0) * 1000
+                if r.status_code in (429, 500, 502, 503, 504):
+                    raise httpx.HTTPStatusError(str(r.status_code), request=r.request, response=r)
+                r.raise_for_status()
+                data = r.json()
+                break
+            except (httpx.TransportError, httpx.HTTPStatusError) as e:
+                last = e
+                time.sleep(delay + random.uniform(0, delay))
+                delay = min(delay * 2, 16)
+        else:
+            return DecisionResponse(answers={k: Answer.failed(q, "transport") for k, q in request.questions.items()},
+                                    latency=LatencyRecord(client_ms=float("nan"), timestamp=time.time()),
+                                    provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id),
+                                    transport_error=str(last))
+        raw_answers = _dig(data, self.answers_path, {}) or {}
+        answers = {k: (self._parse(q, raw_answers[k]) if isinstance(raw_answers.get(k), dict) else Answer.failed(q, "missing answer"))
+                   for k, q in request.questions.items()}
+        in_tok = _first_path(data, self.fields.get("input_tokens", [["usage", "prompt_tokens"], ["usage", "input_tokens"]]))
+        cost = (in_tok / 1e6 * self.price) if (in_tok and self.price) else None
+        return DecisionResponse(
+            answers=answers, latency=LatencyRecord(client_ms=ms, timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id,
+                                    model_id_returned=_first_path(data, self.fields.get("model_returned", [["model"]])),
+                                    generation_id=_first_path(data, self.fields.get("generation_id", [["id"]])),
+                                    billed_input_tokens=in_tok, cost_usd=cost),
+            raw=data,
+        )

sys1bench/adapters/hybrid_router.py

@@ -0,0 +1,49 @@
+"""Hybrid baseline: a System One model decides; below a confidence threshold, escalate to a second
+model (typically an LLM). Reports both the combined answer and which path was taken, so accuracy,
+latency and cost can be plotted against escalation rate. This is the realistic production deployment
+and the comparison vendors' marketing avoids."""
+
+from __future__ import annotations
+
+import time
+
+from ..schemas import DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, get_adapter, register
+
+
+@register("hybrid_router")
+class HybridRouterAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "hybrid", primary: dict | None = None, fallback: dict | None = None,
+                 threshold: float = 0.7, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        primary = dict(primary or {"adapter": "mock"})
+        fallback = dict(fallback or {"adapter": "mock", "skill": 0.95, "latency_ms": 1500.0})
+        self.primary = get_adapter(primary.pop("adapter"), **primary)
+        self.fallback = get_adapter(fallback.pop("adapter"), **fallback)
+        self.threshold = threshold
+        self.tunables.update({"threshold": threshold})
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        p = self.primary.capabilities
+        return p.model_copy(update={"name": f"hybrid({p.name}->{self.fallback.capabilities.name})@{self.threshold}",
+                                    "deployment": "hosted" if "hosted" in (p.deployment, self.fallback.capabilities.deployment) else "local"})
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        first = self.primary.decide(request)
+        low = [k for k, a in first.answers.items() if (not a.ok) or max(a.probs) < self.threshold]
+        if not low:
+            first.raw = {"path": "primary", "escalated": []}
+            return first
+        sub = DecisionRequest(state=request.state, questions={k: request.questions[k] for k in low}, meta=request.meta)
+        second = self.fallback.decide(sub)
+        answers = dict(first.answers)
+        answers.update(second.answers)
+        lat = LatencyRecord(client_ms=first.latency.client_ms + second.latency.client_ms, timestamp=time.time())
+        cost = (first.provider.cost_usd or 0.0) + (second.provider.cost_usd or 0.0)
+        return DecisionResponse(
+            answers=answers, latency=lat,
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.capabilities.name, cost_usd=cost,
+                                    version_hash=f"{first.provider.version_hash}+{second.provider.version_hash}"),
+            raw={"path": "escalated", "escalated": low, "primary": first.raw, "fallback": second.raw},
+        )