sys1bench 0.3.0__py3-none-any.whl

sys1bench/adapters/jev_openrouter.py

@@ -0,0 +1,172 @@
+"""TypeSafe Jev via the OpenRouter alpha Decisions API.
+
+Endpoint: POST https://openrouter.ai/api/alpha/decisions
+Body:     {"model": "typesafe/jev-1.13", "state": <str|obj|list>, "questions": {key: {type, instructions, criteria}}}
+Types:    choice (<=255 options), score (2..10 levels), boolean (our "noul").
+Limits:   32k state tokens, 64k per request, 1200 rpm. Input billed at $0.042/M tokens, output free.
+
+Response parsing is defensive: the alpha API's field names may move. Everything received is stored
+verbatim in `raw`, so re-parsing later never needs another API call. The model id string returned by
+the provider is recorded because `jev-latest` can move silently; pin `jev-1.13`.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from typing import Any
+
+import httpx
+
+from ..schemas import (
+    Answer,
+    DecisionRequest,
+    DecisionResponse,
+    LatencyRecord,
+    ModelCapabilities,
+    ProviderRecord,
+    Question,
+)
+from .base import BaseAdapter, finalize_answer, register
+
+DEFAULT_URL = "https://openrouter.ai/api/alpha/decisions"
+PRICE_PER_M_INPUT = 0.042
+
+
+def _to_vendor_question(q: Question) -> dict[str, Any]:
+    if q.type == "noul":
+        return {"type": "boolean", "instructions": q.instructions}
+    if q.type == "choice":
+        opts = [{"key": c.key, "description": c.description} for c in q.criteria]  # type: ignore[union-attr]
+        if q.allow_abstain:
+            opts.append({"key": q.abstain_key, "description": "None of the listed options applies."})
+        return {"type": "choice", "instructions": q.instructions, "criteria": opts}
+    levels = [{"level": c.level, "description": c.description} for c in q.criteria]  # type: ignore[union-attr]
+    return {"type": "score", "instructions": q.instructions, "criteria": levels}
+
+
+def _first(d: dict, *names: str, default=None):
+    for n in names:
+        if n in d and d[n] is not None:
+            return d[n]
+    return default
+
+
+def parse_answer(q: Question, payload: dict[str, Any]) -> Answer:
+    """Map a vendor answer object to the contract. Handles the shapes seen in public harnesses:
+    choice: {"choice": key, "probabilities": {key: p} | [p...], "confidence": c}
+    score:  {"score"|"expected": x, "distribution"|"probabilities": {...}|[...], "confidence": c}
+    boolean:{"probability"|"p_true"|"value": p, "confidence": c}
+    """
+    keys = q.option_keys
+    conf = _first(payload, "confidence")
+    if q.type == "noul":
+        p = _first(payload, "probability", "p_true", "prob", "value")
+        if isinstance(p, bool):
+            p = 1.0 if p else 0.0
+        if p is None:
+            return Answer.failed(q, "no probability field")
+        return finalize_answer(q, [float(p), 1.0 - float(p)], confidence=conf)
+    dist = _first(payload, "probabilities", "distribution", "probs")
+    if dist is None:
+        return Answer.failed(q, "no distribution field")
+    if isinstance(dist, dict):
+        probs = [float(dist.get(k, dist.get(int(k) if k.isdigit() else k, 0.0))) for k in keys]
+        extra = set(map(str, dist)) - set(keys)
+        abst = q.allow_abstain and q.abstain_key in extra
+    elif isinstance(dist, list):
+        if all(isinstance(x, dict) for x in dist):
+            m = {str(_first(x, "key", "level", "option")): float(_first(x, "probability", "p", "prob")) for x in dist}
+            probs = [m.get(k, 0.0) for k in keys]
+            abst = q.allow_abstain and q.abstain_key in m
+        else:
+            probs = [float(x) for x in dist[: len(keys)]]
+            abst = False
+    else:
+        return Answer.failed(q, f"unparseable distribution {type(dist).__name__}")
+    if abst:
+        # abstain mass is recorded, then folded out so the vector stays over the manifest's options
+        total = sum(probs)
+        if total > 0:
+            probs = [p / total for p in probs]
+    ans = finalize_answer(q, probs, confidence=conf, abstained=bool(abst and max(probs) < 0.5))
+    return ans
+
+
+@register("jev_openrouter")
+class JevOpenRouterAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "typesafe/jev-1.13", url: str = DEFAULT_URL, api_key: str | None = None,
+                 timeout_s: float = 30.0, max_retries: int = 5, route: str = "openrouter", **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        if model_id.endswith("latest"):
+            raise ValueError("pin an explicit Jev version (e.g. typesafe/jev-1.13); 'latest' moves silently")
+        self.url, self.timeout_s, self.max_retries, self.route = url, timeout_s, max_retries, route
+        self.api_key = api_key or os.environ.get("OPENROUTER_API_KEY")
+        self._client = httpx.Client(timeout=timeout_s)
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=self.model_id, deployment="hosted", max_options=255, max_score_levels=10,
+                                 max_state_tokens=32_000, emits_confidence=True, supports_native_abstain=False,
+                                 supports_batching=True, languages=None)
+
+    def _post(self, body: dict[str, Any]) -> tuple[dict[str, Any], float, dict[str, str]]:
+        if not self.api_key:
+            raise RuntimeError("OPENROUTER_API_KEY not set")
+        headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
+        delay = 0.5
+        last_exc: Exception | None = None
+        for attempt in range(self.max_retries):
+            t0 = time.perf_counter()
+            try:
+                r = self._client.post(self.url, json=body, headers=headers)
+                ms = (time.perf_counter() - t0) * 1000
+                if r.status_code in (429, 500, 502, 503, 504):
+                    raise httpx.HTTPStatusError(f"{r.status_code}", request=r.request, response=r)
+                r.raise_for_status()
+                return r.json(), ms, dict(r.headers)
+            except (httpx.TransportError, httpx.HTTPStatusError) as e:
+                last_exc = e
+                time.sleep(delay + random.uniform(0, delay))
+                delay = min(delay * 2, 16.0)
+        raise RuntimeError(f"jev request failed after {self.max_retries} attempts: {last_exc}")
+
+    def parse_raw_answer(self, q: Question, payload: dict) -> Answer:
+        return parse_answer(q, payload)
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        body = {"model": self.model_id, "state": request.state,
+                "questions": {k: _to_vendor_question(q) for k, q in request.questions.items()}}
+        try:
+            data, ms, headers = self._post(body)
+        except Exception as e:
+            return DecisionResponse(
+                answers={k: Answer.failed(q, "transport") for k, q in request.questions.items()},
+                latency=LatencyRecord(client_ms=float("nan"), route=self.route, timestamp=time.time()),
+                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, route=self.route),
+                transport_error=str(e),
+            )
+        raw_answers = _first(data, "answers", "decisions", "results", default={})
+        answers = {}
+        for k, q in request.questions.items():
+            payload = raw_answers.get(k) if isinstance(raw_answers, dict) else None
+            answers[k] = parse_answer(q, payload) if isinstance(payload, dict) else Answer.failed(q, "missing answer")
+        usage = data.get("usage", {}) or {}
+        in_tok = _first(usage, "prompt_tokens", "input_tokens")
+        cost = (in_tok / 1e6) * PRICE_PER_M_INPUT if in_tok else _first(usage, "cost")
+        server_ms = None
+        for h in ("x-processing-ms", "openrouter-processing-ms", "x-response-time"):
+            if h in headers:
+                try:
+                    server_ms = float(str(headers[h]).rstrip("ms"))
+                except ValueError:
+                    pass
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=ms, server_ms=server_ms, route=self.route, timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id,
+                                    model_id_returned=_first(data, "model"), generation_id=_first(data, "id", "generation_id"),
+                                    route=self.route, billed_input_tokens=in_tok, billed_output_tokens=0, cost_usd=cost),
+            raw=data,
+        )

sys1bench/adapters/jev_typesafe.py

@@ -0,0 +1,165 @@
+"""TypeSafe Jev via the first-party API (preferred over the OpenRouter alpha route).
+
+Endpoint: POST https://api.typesafe.ai/v1/systemone   Authorization: Bearer <TypeSafe_API_KEY>
+Body:     {"state": <str|obj|list>, "model": "jev-1.13.0", "questions": {id: Question}}
+Question: choice  -> {"type": "choice", "instructions": str|obj, "criteria": {option: description|null}}   (<=255 options)
+          score   -> {"type": "score",  "instructions": str|obj, "criteria": [level_description, ...]}    (2..10 levels)
+          noul    -> {"type": "noul",   "instructions": str|obj, "criteria": {"true": ..., "false": ...}?}
+Answer:   choice  -> {"type","choice","probabilities":{option: p},"confidence"}
+          score   -> {"type","score" (expected level index),"legend":{"0": desc,...},"probabilities":{"0": p,...},"confidence"}
+          noul    -> {"type","noul": P(yes)}
+Response also carries "model" (versioned id that answered) and "usage": {input_tokens, output_tokens}.
+Limits (models page, 2026-09-22): 64k tokens per request, 32k for state plus longest question, 1200 rpm, 250k tok/s.
+Price: $0.042 per million input tokens; output free.
+
+Score probabilities are keyed by *level index*, so they are mapped back onto the manifest's level values by
+position. Option order in `criteria` is preserved as JSON object order, which is what the permutation suite varies.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from ..schemas import Answer, DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord, Question
+from .base import BaseAdapter, finalize_answer, register
+
+DEFAULT_URL = "https://api.typesafe.ai/v1/systemone"
+PRICE_PER_M_INPUT = 0.042
+ENV_KEYS = ("TypeSafe_API_KEY", "TYPESAFE_API_KEY")
+
+
+def _load_dotenv_key() -> str | None:
+    for k in ENV_KEYS:
+        if os.environ.get(k):
+            return os.environ[k]
+    for candidate in (Path.cwd() / ".env", Path(__file__).resolve().parents[3] / ".env"):
+        if candidate.exists():
+            for line in candidate.read_text().splitlines():
+                line = line.strip()
+                if "=" in line and not line.startswith("#"):
+                    name, _, val = line.partition("=")
+                    if name.strip() in ENV_KEYS and val.strip():
+                        return val.strip().strip('"').strip("'")
+    return None
+
+
+def to_vendor_question(q: Question) -> dict[str, Any]:
+    if q.type == "noul":
+        return {"type": "noul", "instructions": q.instructions}
+    if q.type == "choice":
+        crit: dict[str, Any] = {c.key: (c.description or None) for c in q.criteria}  # type: ignore[union-attr]
+        if q.allow_abstain:
+            crit[q.abstain_key] = "None of the listed options applies."
+        return {"type": "choice", "instructions": q.instructions, "criteria": crit}
+    levels = [(c.description or f"level {c.level}") for c in q.criteria]  # type: ignore[union-attr]
+    return {"type": "score", "instructions": q.instructions, "criteria": levels}
+
+
+def parse_answer(q: Question, payload: dict[str, Any]) -> Answer:
+    keys = q.option_keys
+    if q.type == "noul":
+        p = payload.get("noul")
+        if p is None:
+            return Answer.failed(q, "no noul field")
+        return finalize_answer(q, [float(p), 1.0 - float(p)])
+    dist = payload.get("probabilities")
+    if not isinstance(dist, dict):
+        return Answer.failed(q, "no probabilities map")
+    conf = payload.get("confidence")
+    if q.type == "score":
+        # keyed by level index as string, in the order we sent the levels
+        probs = [float(dist.get(str(i), 0.0)) for i in range(len(keys))]
+        return finalize_answer(q, probs, confidence=conf)
+    probs = [float(dist.get(k, 0.0)) for k in keys]
+    abstained = False
+    if q.allow_abstain and q.abstain_key in dist:
+        p_abst = float(dist[q.abstain_key])
+        abstained = p_abst >= max(probs)
+        s = sum(probs)
+        probs = [p / s for p in probs] if s > 0 else [1.0 / len(keys)] * len(keys)
+    return finalize_answer(q, probs, confidence=conf, abstained=abstained)
+
+
+@register("jev_typesafe")
+class JevTypeSafeAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "jev-1.13.0", url: str = DEFAULT_URL, api_key: str | None = None,
+                 timeout_s: float = 30.0, max_retries: int = 6, allow_alias: bool = False, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        if (model_id.endswith("latest") or model_id.endswith("preview")) and not allow_alias:
+            raise ValueError("pin a versioned Jev id (e.g. jev-1.13.0); aliases move silently. Pass allow_alias=True to override.")
+        self.url, self.timeout_s, self.max_retries = url, timeout_s, max_retries
+        self.api_key = api_key or _load_dotenv_key()
+        self._client = httpx.Client(timeout=timeout_s, http2=False)
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=self.model_id, deployment="hosted", max_options=255, max_score_levels=10,
+                                 max_state_tokens=32_000, emits_confidence=True, supports_native_abstain=False,
+                                 supports_batching=True, languages={"en"})
+
+    def _post(self, body: dict[str, Any]) -> tuple[dict[str, Any], float, dict[str, str]]:
+        if not self.api_key:
+            raise RuntimeError("TypeSafe_API_KEY not set (env or .env)")
+        headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
+        delay, last = 0.5, None
+        for _ in range(self.max_retries):
+            t0 = time.perf_counter()
+            try:
+                r = self._client.post(self.url, json=body, headers=headers)
+                ms = (time.perf_counter() - t0) * 1000
+                if r.status_code in (429, 500, 502, 503, 504, 529):
+                    ra = r.headers.get("retry-after")
+                    if ra:
+                        try:
+                            delay = max(delay, float(ra))
+                        except ValueError:
+                            pass
+                    raise httpx.HTTPStatusError(f"{r.status_code}: {r.text[:200]}", request=r.request, response=r)
+                if 400 <= r.status_code < 500 and r.status_code != 429:
+                    # request rejected (400 over the 32k state limit, 413, 422 malformed): never retry, classify as vendor rejection
+                    raise ValueError(f"{r.status_code} rejected: {r.text[:300]}")
+                r.raise_for_status()
+                return r.json(), ms, dict(r.headers)
+            except (httpx.TransportError, httpx.HTTPStatusError) as e:
+                last = e
+                time.sleep(delay + random.uniform(0, delay))
+                delay = min(delay * 2, 20.0)
+        raise RuntimeError(f"typesafe request failed after {self.max_retries} attempts: {last}")
+
+    def parse_raw_answer(self, q: Question, payload: dict) -> Answer:
+        return parse_answer(q, payload)
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        body = {"state": request.state, "model": self.model_id,
+                "questions": {k: to_vendor_question(q) for k, q in request.questions.items()}}
+        try:
+            data, ms, headers = self._post(body)
+        except Exception as e:
+            kind = "rejected_by_vendor" if isinstance(e, ValueError) else "transport"
+            return DecisionResponse(
+                answers={k: Answer.failed(q, kind) for k, q in request.questions.items()},
+                latency=LatencyRecord(client_ms=float("nan"), route="typesafe", timestamp=time.time()),
+                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, route="typesafe"),
+                transport_error=str(e)[:500],
+            )
+        raw_answers = data.get("answers", {}) or {}
+        answers = {k: (parse_answer(q, raw_answers[k]) if isinstance(raw_answers.get(k), dict) else Answer.failed(q, "missing answer"))
+                   for k, q in request.questions.items()}
+        usage = data.get("usage", {}) or {}
+        in_tok = usage.get("input_tokens")
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=ms, route="typesafe", timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, model_id_returned=data.get("model"),
+                                    version_hash=str(data.get("model") or self.model_id),
+                                    generation_id=headers.get("x-request-id") or headers.get("request-id"), route="typesafe",
+                                    billed_input_tokens=in_tok, billed_output_tokens=usage.get("output_tokens"),
+                                    cost_usd=(in_tok / 1e6 * PRICE_PER_M_INPUT) if in_tok is not None else None),
+            raw=data,
+        )

sys1bench/adapters/laya_local.py

@@ -0,0 +1,192 @@
+"""Convai Laya, run locally (Apache 2.0) through the `laya` package's Router (verified against laya 0.3.4).
+
+    from laya import Router
+    router = Router(preload=True, device="cuda")
+    out = router.predict(state, questions, model=None | "english" | "multilingual" | "typed-decisions")
+
+Question shape matches TypeSafe's: choice criteria is {option: description}, score criteria is an ordered list of
+level descriptions, noul has no criteria. Answers:
+    choice -> {"type","choice","probabilities":{opt: p},"confidence","action":{"act_probability"}}
+    score  -> {"type","score","legend","probabilities":{"0": p,...},"confidence","action":{...}}
+    noul   -> {"type","noul": P(yes),"confidence","action":{...}}
+`action.act_probability` is Laya's act/escalate head; we treat act_probability < 0.5 as a native abstention.
+Result also carries "routing": {"model","repo","reason",...} and "usage".
+
+Tunables: `head_max_len` and `max_len` are set on the checkpoint's `agent.cfg` before inference (Suite C ablates them;
+the 77-label collapse at the default budget is a configuration effect). Checkpoint defaults: english 512/192,
+multilingual 1024/256, typed-decisions 1024/256. The package prints a warning and falls back to CPU when CUDA is
+unavailable or out of memory; we record the device actually used in `hardware` so latency tables stay honest.
+"""
+
+from __future__ import annotations
+
+import platform
+import time
+from typing import Any
+
+from ..schemas import Answer, DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord, Question
+from .base import BaseAdapter, FatalAdapterError, finalize_answer, register
+
+CHECKPOINTS = {"english": ("convaiinnovations/laya", 512, 192), "multilingual": ("convaiinnovations/laya/multilingual", 1024, 256),
+               "typed-decisions": ("convaiinnovations/laya/typed-decisions", 1024, 256), "router": ("convaiinnovations/laya", 1024, 256)}
+ALIASES = {"laya": "english", "laya-multilingual": "multilingual", "laya-typed-decisions": "typed-decisions"}
+
+
+def to_vendor_question(q: Question) -> dict[str, Any]:
+    if q.type == "noul":
+        return {"type": "noul", "instructions": q.instructions}
+    if q.type == "choice":
+        crit = {c.key: (c.description or c.key) for c in q.criteria}  # type: ignore[union-attr]
+        if q.allow_abstain:
+            crit[q.abstain_key] = "None of the listed options applies."
+        return {"type": "choice", "instructions": q.instructions, "criteria": crit}
+    return {"type": "score", "instructions": q.instructions,
+            "criteria": [(c.description or f"level {c.level}") for c in q.criteria]}  # type: ignore[union-attr]
+
+
+def parse_answer(q: Question, payload: dict[str, Any], abstain_threshold: float = 0.5) -> Answer:
+    keys = q.option_keys
+    conf = payload.get("confidence")
+    act = (payload.get("action") or {}).get("act_probability")
+    abstained = act is not None and float(act) < abstain_threshold
+    if q.type == "noul":
+        p = payload.get("noul")
+        if p is None:
+            return Answer.failed(q, "no noul field")
+        a = finalize_answer(q, [float(p), 1.0 - float(p)], confidence=None, abstained=abstained)
+    else:
+        dist = payload.get("probabilities")
+        if not isinstance(dist, dict):
+            return Answer.failed(q, "no probabilities map")
+        if q.type == "score":
+            probs = [float(dist.get(str(i), 0.0)) for i in range(len(keys))]
+        else:
+            probs = [float(dist.get(k, 0.0)) for k in keys]
+            if q.allow_abstain and q.abstain_key in dist:
+                abstained = abstained or float(dist[q.abstain_key]) >= max(probs)
+                s = sum(probs)
+                probs = [x / s for x in probs] if s > 0 else [1.0 / len(keys)] * len(keys)
+        a = finalize_answer(q, probs, confidence=conf, abstained=abstained)
+    if a.ok and act is not None:
+        a = a.model_copy(update={"raw_prob_sum": a.raw_prob_sum})
+    return a
+
+
+@register("laya_local")
+class LayaLocalAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "convaiinnovations/laya", checkpoint: str = "english", device: str = "cuda",
+                 head_max_len: int | None = None, max_len: int | None = None, batch_size: int = 1,
+                 hardware: str | None = None, abstain_threshold: float = 0.5, strict_device: bool = True, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        checkpoint = ALIASES.get(checkpoint, checkpoint)
+        if checkpoint not in CHECKPOINTS:
+            raise ValueError(f"unknown Laya checkpoint {checkpoint!r}; known: {sorted(CHECKPOINTS)} (+ aliases {sorted(ALIASES)})")
+        self.checkpoint, self.device, self.batch_size, self.abstain_threshold = checkpoint, device, batch_size, abstain_threshold
+        self.strict_device = strict_device
+        _, dflt_max, dflt_head = CHECKPOINTS[checkpoint]
+        self.head_max_len = head_max_len or dflt_head
+        self.max_len = max_len or dflt_max
+        self.hardware = hardware
+        self.tunables.update({"head_max_len": self.head_max_len, "max_len": self.max_len, "checkpoint": checkpoint})
+        self._router = None
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=f"laya:{self.checkpoint}", deployment="local", max_options=255,
+                                 max_state_tokens=self.max_len - self.head_max_len, emits_confidence=True,
+                                 supports_native_abstain=True, supports_batching=True,
+                                 languages={"en"} if self.checkpoint == "english" else None,
+                                 tunables={"head_max_len": self.head_max_len, "max_len": self.max_len})
+
+    def _load(self):
+        if self._router is not None:
+            return self._router
+        try:
+            from laya import Router  # type: ignore
+        except ImportError as e:  # pragma: no cover
+            raise RuntimeError("install Laya: pip install laya (github.com/NandhaKishorM/laya)") from e
+        # Lazy router: load only the checkpoint we evaluate (preloading all three needs ~6 GB and, on a shared GPU,
+        # silently lands on CPU). "router" mode keeps the full preload because it dispatches per language.
+        if self.checkpoint == "router":
+            router = Router(preload=True, device=self.device)
+            agents = list((getattr(router, "_agents", {}) or {}).values())
+        else:
+            router = Router(preload=False, device=self.device)
+            agents = [router.load(self.checkpoint)]
+        for ag in agents:
+            if hasattr(ag, "cfg"):
+                ag.cfg["head_max_len"] = self.head_max_len
+                ag.cfg["max_len"] = self.max_len
+            dev = str(getattr(ag, "device", ""))
+            if self.device.startswith("cuda") and "cuda" not in dev and hasattr(ag, "model"):
+                try:  # the package falls back to CPU when CUDA context creation fails at import; retry the move
+                    import torch  # type: ignore
+
+                    ag.model.to("cuda")
+                    ag.device = torch.device("cuda") if hasattr(torch, "device") else "cuda"
+                except Exception:
+                    pass
+            dev = str(getattr(ag, "device", ""))
+            if self.device.startswith("cuda") and "cuda" not in dev and self.strict_device:
+                del router, agents  # release the half-loaded model before aborting
+                raise FatalAdapterError(f"Laya checkpoint {self.checkpoint!r} loaded on {dev or 'cpu'} although device={self.device!r} was requested; "
+                                        "refusing to record CPU latency as GPU. Free GPU memory or pass strict_device=false.")
+        ag = agents[0] if agents else None
+        dev = str(getattr(ag, "device", self.device)) if ag is not None else self.device
+        if self.hardware is None:
+            gpu = None
+            try:
+                import torch  # type: ignore
+
+                if "cuda" in dev and torch.cuda.is_available():
+                    gpu = torch.cuda.get_device_name(0)
+            except Exception:  # pragma: no cover
+                pass
+            self.hardware = f"{gpu} ({dev})" if gpu else f"CPU {platform.machine()} ({dev})"
+        self._router = router  # only after every check passed
+        return self._router
+
+    def parse_raw_answer(self, q: Question, payload: dict) -> Answer:
+        return parse_answer(q, payload, self.abstain_threshold)
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        router = self._load()
+        vq = {k: to_vendor_question(q) for k, q in request.questions.items()}
+        model = None if self.checkpoint == "router" else self.checkpoint
+        t0 = time.perf_counter()
+        try:
+            out = router.predict(request.state, vq, model=model)
+        except Exception as e:
+            # laya raises ValueError("... options exceed head_max_len=N") when the option strings do not fit the budget:
+            # that is the model refusing the request, not a harness bug, so it is classified like a vendor 4xx.
+            kind = "rejected_by_model" if isinstance(e, ValueError) else "adapter_exception"
+            return DecisionResponse(
+                answers={k: Answer.failed(q, kind) for k, q in request.questions.items()},
+                latency=LatencyRecord(client_ms=float("nan"), timestamp=time.time()),
+                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, hardware=self.hardware),
+                transport_error=repr(e)[:500])
+        ms = (time.perf_counter() - t0) * 1000
+        raw_answers = out.get("answers", {}) if isinstance(out, dict) else {}
+        # laya truncates silently to max_len - head_max_len state tokens and reports nothing; estimate it (~4 chars/token)
+        from ..generators.base import estimate_tokens
+
+        est_tokens = estimate_tokens(request.state_text())
+        truncated = bool(out.get("truncated", False)) if isinstance(out, dict) else False
+        truncated = truncated or est_tokens > (self.max_len - self.head_max_len)
+        answers = {}
+        for k, q in request.questions.items():
+            a = parse_answer(q, raw_answers[k], self.abstain_threshold) if isinstance(raw_answers.get(k), dict) else Answer.failed(q, "missing answer")
+            if a.ok and truncated:
+                a = a.model_copy(update={"truncated": True})
+            answers[k] = a
+        routing = out.get("routing", {}) if isinstance(out, dict) else {}
+        usage = out.get("usage", {}) if isinstance(out, dict) else {}
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=ms, compute_ms=ms, batch_size=self.batch_size, timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id,
+                                    model_id_returned=f"laya:{routing.get('model', self.checkpoint)}",
+                                    version_hash=f"{routing.get('repo', '')}|head{self.head_max_len}|max{self.max_len}",
+                                    hardware=self.hardware, billed_input_tokens=usage.get("input_tokens"), cost_usd=0.0),
+            raw=out if isinstance(out, dict) else {"out": str(out)},
+        )

sys1bench/adapters/llm_constrained.py

@@ -0,0 +1,104 @@
+"""Constrained-decoding LLM baseline with first-token logprobs, so LLMs get calibration metrics.
+
+Backend A (default): any OpenAI-compatible server (vLLM, OpenRouter) with `logprobs`. The prompt lists
+options as single-letter keys; probabilities are the renormalised logprobs of the first generated token
+over the option letters, which is the standard "logit-based" calibration protocol for MCQ.
+Backend B: vLLM + Outlines regex-constrained generation (import guarded); same probability extraction.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import string
+import time
+
+import httpx
+
+from ..schemas import (
+    Answer,
+    DecisionRequest,
+    DecisionResponse,
+    LatencyRecord,
+    ModelCapabilities,
+    ProviderRecord,
+    Question,
+)
+from .base import BaseAdapter, finalize_answer, register
+
+LETTERS = string.ascii_uppercase + string.ascii_lowercase + string.digits
+
+
+def build_prompt(state: str, q: Question) -> tuple[str, list[str]]:
+    keys = q.option_keys
+    letters = list(LETTERS[: len(keys)])
+    if q.type == "noul":
+        lines = ["A. yes", "B. no"]
+    elif q.type == "choice":
+        lines = [f"{L}. {c.key}: {c.description}" for L, c in zip(letters, q.criteria)]  # type: ignore[union-attr]
+    else:
+        lines = [f"{L}. level {c.level}: {c.description}" for L, c in zip(letters, q.criteria)]  # type: ignore[union-attr]
+    prompt = (f"State:\n{state}\n\nQuestion: {q.instructions}\nOptions:\n" + "\n".join(lines) +
+              "\n\nAnswer with the single letter of the best option.\nAnswer:")
+    return prompt, letters
+
+
+@register("llm_constrained")
+class LLMConstrainedAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "Qwen/Qwen2.5-3B-Instruct", base_url: str = "http://localhost:8000/v1",
+                 api_key_env: str = "OPENAI_API_KEY", top_logprobs: int = 20, price_per_m_input: float | None = None,
+                 price_per_m_output: float | None = None, timeout_s: float = 60.0, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.base_url, self.top_logprobs = base_url.rstrip("/"), top_logprobs
+        self.key = os.environ.get(api_key_env, "EMPTY")
+        self.pi, self.po = price_per_m_input, price_per_m_output
+        self._client = httpx.Client(timeout=timeout_s)
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=f"llm:{self.model_id}", deployment="hosted" if "localhost" not in self.base_url else "local",
+                                 max_options=len(LETTERS), emits_confidence=False, supports_batching=False, deterministic=False)
+
+    def _one(self, state: str, q: Question) -> tuple[Answer, dict]:
+        prompt, letters = build_prompt(state, q)
+        body = {"model": self.model_id, "prompt": prompt, "max_tokens": 1, "temperature": 0.0,
+                "logprobs": self.top_logprobs}
+        r = self._client.post(f"{self.base_url}/completions", json=body,
+                              headers={"Authorization": f"Bearer {self.key}"})
+        r.raise_for_status()
+        data = r.json()
+        lp = data["choices"][0].get("logprobs", {})
+        top = (lp.get("top_logprobs") or [{}])[0]
+        scores = []
+        for L in letters:
+            cands = [v for k, v in top.items() if k.strip() == L]
+            scores.append(max(cands) if cands else -30.0)
+        m = max(scores)
+        w = [math.exp(s - m) for s in scores]
+        z = sum(w)
+        probs = [x / z for x in w]
+        return finalize_answer(q, probs), data
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = time.perf_counter()
+        answers, raws, in_tok, out_tok = {}, {}, 0, 0
+        err = None
+        for k, q in request.questions.items():
+            try:
+                a, data = self._one(request.state_text(), q)
+                answers[k] = a
+                raws[k] = data
+                u = data.get("usage", {})
+                in_tok += u.get("prompt_tokens", 0)
+                out_tok += u.get("completion_tokens", 0)
+            except Exception as e:  # transport or parse
+                answers[k] = Answer.failed(q, "transport")
+                err = str(e)
+        ms = (time.perf_counter() - t0) * 1000
+        cost = None
+        if self.pi is not None:
+            cost = in_tok / 1e6 * self.pi + out_tok / 1e6 * (self.po or 0.0)
+        return DecisionResponse(answers=answers, latency=LatencyRecord(client_ms=ms, timestamp=time.time()),
+                                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id,
+                                                        billed_input_tokens=in_tok, billed_output_tokens=out_tok, cost_usd=cost),
+                                raw=raws, transport_error=err)