llm2decision 0.1.0__py3-none-any.whl

llm2decision/__init__.py

@@ -0,0 +1,20 @@
+"""Any hosted LLM as a decision model, used like the TypeSafe (Jev) SDK.
+
+    import llm2decision as l2d
+
+    with l2d.DecisionClient("qwen3.8-27b@nebius") as client:
+        r = client.system_one(state, {"team": l2d.Choice(criteria={"billing": "…", "other": None})})
+        r.choices["team"].choice
+"""
+from .client import AsyncDecisionClient, DecisionClient
+from .config import LLM2DecisionWarning, ModelInfo, bindings, list_models, providers
+from .errors import (AuthError, ConfigError, LLM2DecisionError, MarksError, QuestionError, TransportError,
+                     UnreadableAnswer)
+from .marks import Marks, assign, read
+from .types import (Answer, Choice, ChoiceAnswer, Meta, Noul, NoulAnswer, Score, ScoreAnswer, SystemOneResponse,
+                    Tfu, TfuAnswer, Usage)
+
+__all__ = ["DecisionClient", "AsyncDecisionClient", "list_models", "ModelInfo", "Noul", "Tfu", "Choice", "Score", "NoulAnswer", "TfuAnswer", "ChoiceAnswer",
+           "ScoreAnswer", "Answer", "SystemOneResponse", "Usage", "Meta", "LLM2DecisionError", "QuestionError",
+           "ConfigError", "AuthError", "TransportError", "UnreadableAnswer", "LLM2DecisionWarning", "bindings", "providers", "Marks", "MarksError", "assign", "read"]
+__version__ = "0.1.0"

llm2decision/__main__.py

@@ -0,0 +1,63 @@
+"""`python -m llm2decision` (or `llm2decision`): list the known models, check one, or measure it.
+
+    llm2decision models [--provider nebius]
+    llm2decision check qwen3.8-27b@nebius [--save]
+    llm2decision check Qwen/Qwen3-8B --base-url http://localhost:8000/v1
+    llm2decision bench qwen3.8-27b@nebius [--data test.jsonl] [--limit 200] [--out answers.jsonl]
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+
+from . import bench
+from .client import DecisionClient
+from .config import list_models
+from .errors import LLM2DecisionError
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="llm2decision")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    m = sub.add_parser("models", help="the known bindings")
+    m.add_argument("--provider")
+    c = sub.add_parser("check", help="probe a model through its provider")
+    b = sub.add_parser("bench", help="measure a model on Decision Questions")
+    for p in (c, b):
+        p.add_argument("model")
+        p.add_argument("--provider")
+        p.add_argument("--base-url")
+        p.add_argument("--key-env", help="the environment variable holding the key")
+        p.add_argument("--env-file")
+    c.add_argument("--save", action="store_true", help="record a pass in the user's models.json")
+    b.add_argument("--data", help="a Decision Questions file (default: the published set, downloaded once)")
+    b.add_argument("--out", help="where answers are kept and resumed from (default: bench_<model>.jsonl)")
+    b.add_argument("--limit", type=int, help="the first N questions only")
+    b.add_argument("--workers", type=int, default=8)
+    a = ap.parse_args(argv)
+    if a.cmd == "models":
+        for x in list_models(a.provider):
+            cal = ",".join(x.calibrated) or "-"
+            print(f"{x.name:<34} {x.transport:<14} calibrated={cal:<34} checked={x.checked or '-'}")
+        return 0
+    try:
+        with DecisionClient(a.model, provider=a.provider, base_url=a.base_url, key_env=a.key_env,
+                            env_file=a.env_file, **({"workers": 1} if a.cmd == "bench" else {})) as client:
+            if a.cmd == "check":
+                rep = client.check(save=a.save)
+                print(rep)
+                return 0 if rep.ok else 1
+            items = bench.load(a.data)[: a.limit]
+            out = a.out or f"bench_{re.sub(r'[^A-Za-z0-9._@-]+', '_', a.model)}.jsonl"
+            rows = bench.run(client, items, out, workers=a.workers,
+                             progress=lambda s: print(s, file=sys.stderr, flush=True))
+            print(bench.report(rows))
+            return 0 if len(rows) == len(items) else 1
+    except LLM2DecisionError as e:
+        print(f"llm2decision: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

llm2decision/_http.py

@@ -0,0 +1,127 @@
+"""JSON over HTTP with the standard library: one kept-alive connection per thread and host, proxies
+from the environment (`HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY`), no dependencies.
+
+Network failures raise `NetworkError` (the transport retries them); an HTTP status is returned as is.
+A kept-alive connection the server has closed in the meantime is reopened once, silently.
+"""
+from __future__ import annotations
+
+import base64
+import http.client
+import json
+import ssl
+import threading
+import urllib.request
+from dataclasses import dataclass
+from email.message import Message
+from typing import Any
+from urllib.parse import unquote, urlsplit
+
+
+class NetworkError(Exception):
+    """The request did not get an HTTP response: refused, reset, timed out, name not resolved."""
+
+
+@dataclass
+class Response:
+    status: int
+    headers: Message
+    text: str
+
+    def json(self) -> Any:
+        return json.loads(self.text)
+
+
+def _proxy_for(scheme: str, host: str):
+    """The proxy URL the environment names for this scheme and host, unless `NO_PROXY` exempts the host."""
+    proxies = urllib.request.getproxies()
+    url = proxies.get(scheme)
+    if not url or urllib.request.proxy_bypass(host):
+        return None
+    return urlsplit(url if "://" in url else f"http://{url}")
+
+
+def _proxy_auth(p) -> dict[str, str]:
+    if not p.username:
+        return {}
+    token = base64.b64encode(f"{unquote(p.username)}:{unquote(p.password or '')}".encode()).decode()
+    return {"Proxy-Authorization": f"Basic {token}"}
+
+
+class Session:
+    """Default headers plus per-thread connections. Thread-safe: threads never share a connection."""
+
+    def __init__(self, headers: dict[str, str] | None = None):
+        self.headers = dict(headers or {})
+        self._local = threading.local()
+        self._lock = threading.Lock()
+        self._open: list[http.client.HTTPConnection] = []
+        self._ssl = ssl.create_default_context()
+
+    def _connection(self, scheme: str, host: str, port: int, timeout: float):
+        conns = self._local.__dict__.setdefault("conns", {})
+        key = (scheme, host, port)
+        conn = conns.get(key)
+        fresh = conn is None
+        if fresh:
+            proxy = _proxy_for(scheme, host)
+            if scheme == "https":
+                if proxy:
+                    conn = http.client.HTTPSConnection(proxy.hostname, proxy.port or 8080, timeout=timeout,
+                                                       context=self._ssl)
+                    conn.set_tunnel(host, port, headers=_proxy_auth(proxy))
+                else:
+                    conn = http.client.HTTPSConnection(host, port, timeout=timeout, context=self._ssl)
+            else:
+                target = proxy if proxy else None
+                conn = http.client.HTTPConnection(target.hostname if target else host,
+                                                  (target.port or 8080) if target else port, timeout=timeout)
+                conn._llm2decision_proxy = proxy           # an absolute URL in the request line
+            conns[key] = conn
+            with self._lock:
+                self._open.append(conn)
+        conn.timeout = timeout
+        if conn.sock is not None:
+            conn.sock.settimeout(timeout)
+        return conn, fresh
+
+    def _drop(self, scheme: str, host: str, port: int) -> None:
+        conn = self._local.__dict__.get("conns", {}).pop((scheme, host, port), None)
+        if conn is not None:
+            conn.close()
+
+    def post(self, url: str, body: Any, *, timeout: float, headers: dict[str, str] | None = None) -> Response:
+        u = urlsplit(url)
+        scheme, host = u.scheme, u.hostname or ""
+        port = u.port or (443 if scheme == "https" else 80)
+        path = (u.path or "/") + (f"?{u.query}" if u.query else "")
+        data = json.dumps(body).encode("utf-8")
+        hdrs = {**self.headers, **(headers or {}), "Content-Type": "application/json",
+                "Content-Length": str(len(data)), "Accept": "application/json"}
+        for _ in range(2):
+            conn, fresh = self._connection(scheme, host, port, timeout)
+            target = url if getattr(conn, "_llm2decision_proxy", None) else path
+            if getattr(conn, "_llm2decision_proxy", None):
+                hdrs.update(_proxy_auth(conn._llm2decision_proxy))
+            try:
+                conn.request("POST", target, body=data, headers=hdrs)
+                r = conn.getresponse()
+                raw = r.read()
+            except (http.client.RemoteDisconnected, BrokenPipeError, ConnectionResetError) as e:
+                self._drop(scheme, host, port)
+                if fresh:                                   # a new connection failed: a real network error
+                    raise NetworkError(type(e).__name__) from e
+                continue                                    # a kept-alive one went stale: reopen once
+            except (OSError, http.client.HTTPException) as e:
+                self._drop(scheme, host, port)
+                raise NetworkError(type(e).__name__) from e
+            if (r.getheader("Connection") or "").lower() == "close":
+                self._drop(scheme, host, port)
+            return Response(r.status, r.headers, raw.decode("utf-8", "replace"))
+        raise NetworkError("connection closed")              # pragma: no cover - two stale connections
+
+    def close(self) -> None:
+        with self._lock:
+            conns, self._open = self._open, []
+        for c in conns:
+            c.close()

llm2decision/auth.py

@@ -0,0 +1,144 @@
+"""Where a provider's key comes from and how it is sent — with nothing printed and nothing sent
+where it does not belong.
+
+Sources, first found wins:
+
+  1. `key=` in code — a string, or a callable returning one (a keyring, a vault, a secrets manager)
+  2. `key_env=` — the name of an environment variable chosen by the caller
+  3. the provider's own variable from its configuration (`NEBIUS_API_KEY`, `OPENROUTER_API_KEY`, …)
+  4. an env file — only when named: `env_file=` or `LLM2DECISION_ENV_FILE`; never picked up silently,
+     and it never overrides a variable already set in the environment
+
+A provider configured with `auth: none` (a vLLM, llama-server or Ollama of your own) needs no key.
+
+A key found in the environment or an env file (3, 4) is sent only to the provider's own host: with
+`base_url` pointed elsewhere it is withheld, and only an explicit `key=` or `key_env=` is sent there.
+A typo in an address must not hand a key to a stranger.
+
+How it is sent comes from the provider: `Authorization: Bearer <key>` (OpenAI-style), `x-api-key`
+(Anthropic), `api-key` (Azure) or any header with an optional scheme.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Mapping
+from urllib.parse import urlparse
+
+ENV_FILE_VAR = "LLM2DECISION_ENV_FILE"
+
+
+from .errors import AuthError  # noqa: E402  re-exported
+
+
+@dataclass(frozen=True)
+class AuthSpec:
+    """How a provider takes its key (from the provider's configuration)."""
+    mode: str = "header"                 # "header" or "none"
+    header: str = "Authorization"
+    scheme: str = "Bearer"               # "" for a bare key, as `x-api-key` takes it
+    env: str = ""                        # the provider's own variable, e.g. NEBIUS_API_KEY
+    host: str = ""                       # the provider's own host: environment keys go only here
+
+    @classmethod
+    def from_config(cls, cfg: Mapping | None, base_url: str = "") -> "AuthSpec":
+        cfg = dict(cfg or {})
+        if cfg.get("mode") == "none" or cfg.get("type") == "none":
+            return cls(mode="none", host=_host(base_url))
+        return cls(mode="header", header=cfg.get("header", "Authorization"),
+                   scheme=cfg.get("scheme", "Bearer"), env=cfg.get("env", ""),
+                   host=cfg.get("host") or _host(base_url))
+
+
+@dataclass(frozen=True)
+class Credential:
+    """A resolved key and where it came from. `repr` and `str` never show the key."""
+    value: str | None
+    source: str                          # "code", "env:NAME", "env_file:PATH:NAME", "none"
+    explicit: bool                       # given by the caller (code, key_env) rather than found
+
+    def __repr__(self) -> str:
+        return f"Credential(source={self.source!r}, set={self.value is not None})"
+
+    __str__ = __repr__
+
+
+def _host(url: str) -> str:
+    return (urlparse(url).hostname or "").lower()
+
+
+def parse_env_file(path: str | Path) -> dict[str, str]:
+    """`NAME=value` lines: `export ` allowed, `#` comments and blank lines skipped, one level of
+    matching quotes removed, an inline ` #` comment cut from unquoted values."""
+    out: dict[str, str] = {}
+    for raw in Path(path).read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("export "):
+            line = line[len("export "):].lstrip()
+        if "=" not in line:
+            continue
+        name, value = line.split("=", 1)
+        name, value = name.strip(), value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
+            value = value[1:-1]
+        elif " #" in value:
+            value = value.split(" #", 1)[0].rstrip()
+        if name:
+            out[name] = value
+    return out
+
+
+def resolve(spec: AuthSpec, *, key: str | Callable[[], str] | None = None, key_env: str | None = None,
+            env_file: str | Path | None = None, base_url: str = "",
+            environ: Mapping[str, str] | None = None) -> Credential:
+    """The key for one provider, by the order in the module docstring."""
+    environ = os.environ if environ is None else environ
+    if spec.mode == "none" and key is None and key_env is None:
+        return Credential(None, "none", False)
+    if key is not None:
+        value = key() if callable(key) else key
+        if not isinstance(value, str) or not value.strip():
+            raise AuthError("the key given in code is empty")
+        return Credential(value.strip(), "code", True)
+    if key_env:
+        value = environ.get(key_env, "").strip()
+        if not value:
+            raise AuthError(f"key_env names {key_env!r}, which is not set or is empty")
+        return Credential(value, f"env:{key_env}", True)
+    looked = []
+    if spec.env:
+        looked.append(f"the environment variable {spec.env}")
+        value = environ.get(spec.env, "").strip()
+        if value:
+            return _for_host(Credential(value, f"env:{spec.env}", False), spec, base_url)
+    file = env_file or environ.get(ENV_FILE_VAR)
+    if file and spec.env:
+        looked.append(f"{spec.env} in the env file {file}")
+        p = Path(file)
+        if not p.is_file():
+            raise AuthError(f"env file {file} does not exist")
+        value = parse_env_file(p).get(spec.env, "").strip()
+        if value:
+            return _for_host(Credential(value, f"env_file:{file}:{spec.env}", False), spec, base_url)
+    where = ", ".join(looked) or "nowhere: the provider names no variable"
+    raise AuthError(f"no key for this provider; looked in {where}. Pass key=…, key_env=…, or "
+                    f"env_file=… (or set {ENV_FILE_VAR})")
+
+
+def _for_host(cred: Credential, spec: AuthSpec, base_url: str) -> Credential:
+    """A key found rather than given goes only to the provider's own host."""
+    target = _host(base_url) or spec.host
+    if spec.host and target and target != spec.host:
+        raise AuthError(f"a key from {cred.source.split(':')[0]} is meant for {spec.host}, not {target}; "
+                        "pass key=… or key_env=… to send a key to another address")
+    return cred
+
+
+def headers(spec: AuthSpec, cred: Credential) -> dict[str, str]:
+    """The request headers that carry the key."""
+    if cred.value is None:
+        return {}
+    return {spec.header: f"{spec.scheme} {cred.value}" if spec.scheme else cred.value}

llm2decision/bench.py

@@ -0,0 +1,189 @@
+"""Measure a model on Decision Questions, the set behind the README's model table.
+
+    llm2decision bench qwen3.8-27b@nebius                     # the published set, downloaded once
+    llm2decision bench my-qwen@local --data test.jsonl --limit 200
+
+Every question goes out on its own, as in the table. Answers are appended to `--out` as they come, so an
+interrupted run continues where it stopped when started again. The report gives accuracy over the
+answered questions (all and per type), the share left undecided, and the calibration error when the
+model returns probabilities.
+"""
+from __future__ import annotations
+
+import json
+import os
+import signal
+import sys
+import threading
+import time
+import urllib.request
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .errors import LLM2DecisionError, TransportError
+
+# the revision the README's model table was measured on; the cache is per revision, so a newer one is fetched anew
+DATA_REVISION = "04da4c4131f23424b5d41e219459fa1ce5a8fa69"
+DATA_URL = f"https://huggingface.co/datasets/mihailgribov/decision-questions/resolve/{DATA_REVISION}/data/test.jsonl"
+TYPES = ("noul", "tfu", "choice", "score")
+NAMES = {"noul": "yes/no", "tfu": "yes/no/unknown", "choice": "choice", "score": "score"}
+
+
+def cache_dir() -> Path:
+    return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "llm2decision"
+
+
+def load(path: str | os.PathLike | None = None) -> list[dict]:
+    """The items of a Decision Questions file; without a path, the published set (downloaded once and cached)."""
+    if path is None:
+        path = cache_dir() / f"decision_questions_test_{DATA_REVISION[:12]}.jsonl"
+        if not path.exists():
+            path.parent.mkdir(parents=True, exist_ok=True)
+            print(f"downloading Decision Questions to {path}", file=sys.stderr, flush=True)
+            tmp = path.with_suffix(".part")
+            with urllib.request.urlopen(DATA_URL, timeout=120) as r, open(tmp, "wb") as f:
+                f.write(r.read())
+            tmp.replace(path)
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def question(item: dict) -> dict:
+    q = dict(item["question"])
+    if isinstance(q.get("criteria"), str):
+        q["criteria"] = json.loads(q["criteria"])
+    return q
+
+
+def state(item: dict):
+    return json.loads(item["state"]) if item.get("state_format") == "json" else item["state"]
+
+
+def ask(client, item: dict) -> dict:
+    """One item, one call; the answer as probabilities over the item's answer labels."""
+    a = client.system_one(state(item), {"q": question(item)}).answers["q"]
+    if item["type"] == "noul":
+        probs = {"yes": a.noul, "no": 1 - a.noul}
+    else:
+        probs = {str(k): v for k, v in a.probabilities.items()}
+    return {"id": item["id"], "type": item["type"], "answer": str(item["answer"]),
+            "probs": {k: round(v, 6) for k, v in probs.items()},
+            "answered": a.meta.answered, "logprobs": a.meta.logprobs}
+
+
+def _done(out: Path) -> dict[str, dict]:
+    if not out.exists():
+        return {}
+    rows = {}
+    for line in out.read_text(encoding="utf-8").splitlines():
+        if line.strip():
+            r = json.loads(line)
+            rows[r["id"]] = r
+    return rows
+
+
+def run(client, items: list[dict], out: str | os.PathLike, *, workers: int = 8, progress=print) -> dict[str, dict]:
+    """Ask what `out` does not hold yet, appending each answer as it comes; returns every row of `out`
+    for the given items. Ctrl-C stops after the requests in flight; a refused request stops the run."""
+    out = Path(out)
+    rows = _done(out)
+    todo = [x for x in items if x["id"] not in rows]
+    progress(f"{len(items) - len(todo)}/{len(items)} already in {out}, {len(todo)} to ask")
+    stop = threading.Event()
+    previous = None
+    if threading.current_thread() is threading.main_thread():
+        def on_signal(sig, _):
+            stop.set()
+            progress("stopping: finishing the requests in flight…")
+        previous = signal.signal(signal.SIGINT, on_signal)
+    errors, t0, n0 = 0, time.time(), len(rows)
+    try:
+        with open(out, "a", encoding="utf-8") as f, ThreadPoolExecutor(workers) as ex:
+            it, running = iter(todo), {}
+            while True:
+                while not stop.is_set() and len(running) < workers:
+                    x = next(it, None)
+                    if x is None:
+                        break
+                    running[ex.submit(ask, client, x)] = x
+                if not running:
+                    break
+                for fut in wait(running, timeout=1, return_when=FIRST_COMPLETED)[0]:
+                    x = running.pop(fut)
+                    try:
+                        r = fut.result()
+                    except LLM2DecisionError as e:
+                        errors += 1
+                        progress(f"  {x['id']}: {str(e)[:160]}")
+                        if isinstance(e, TransportError) and "HTTP 4" in str(e) and "429" not in str(e):
+                            stop.set()                  # a refusal (key, balance, model): the rest would fail too
+                        continue
+                    f.write(json.dumps(r, ensure_ascii=False) + "\n")
+                    f.flush()
+                    rows[r["id"]] = r
+                    done = len(rows) - n0
+                    if done % 50 == 0:
+                        rate = done / max(time.time() - t0, 1e-9)
+                        progress(f"  {len(rows)}/{len(items)}  {rate:.1f}/s  errors {errors}")
+    finally:
+        if previous is not None:
+            signal.signal(signal.SIGINT, previous)
+    wanted = {x["id"] for x in items}
+    rows = {k: v for k, v in rows.items() if k in wanted}
+    progress(f"{len(rows)}/{len(items)} answered and saved, errors {errors}")
+    if len(rows) < len(items):
+        progress("to continue: run the same command again")
+    return rows
+
+
+def _right(r: dict) -> bool:
+    return max(r["probs"], key=r["probs"].get) == r["answer"]
+
+
+def _ece(rows: list[dict], bins: int) -> float:
+    e = 0.0
+    for b in range(bins):
+        inside = [r for r in rows if b / bins < max(r["probs"].values()) <= (b + 1) / bins]
+        if inside:
+            conf = sum(max(r["probs"].values()) for r in inside) / len(inside)
+            acc = sum(map(_right, inside)) / len(inside)
+            e += len(inside) / len(rows) * abs(acc - conf)
+    return e
+
+
+def ece(rows: list[dict], bins: int = 10) -> float:
+    """Expected calibration error of the top answer's probability over the answered rows: per question
+    type, then weighted by the type's share, so that easy types do not hide a badly calibrated one."""
+    rows = [r for r in rows if r["answered"]]
+    by = [[r for r in rows if r["type"] == t] for t in TYPES]
+    return sum(len(g) * _ece(g, bins) for g in by if g) / len(rows) if rows else 0.0
+
+
+@dataclass
+class Report:
+    n: int
+    accuracy: float | None
+    by_type: dict[str, float | None] = field(default_factory=dict)
+    undecided: float = 0.0
+    calibration_error: float | None = None       # None: the model answers in text
+
+    def __str__(self) -> str:
+        f = lambda v: "-" if v is None else f"{v:.3f}"
+        lines = [f"questions        {self.n}", f"accuracy         {f(self.accuracy)}  (over answered questions)"]
+        lines += [f"  {NAMES[t]:<15}{f(v)}" for t, v in self.by_type.items()]
+        lines.append(f"undecided        {100 * self.undecided:.1f}%")
+        lines.append("calibration err  " + ("no probabilities (text answers)" if self.calibration_error is None
+                                            else f(self.calibration_error)))
+        return "\n".join(lines)
+
+
+def report(rows) -> Report:
+    rows = list(rows.values() if isinstance(rows, dict) else rows)
+    answered = [r for r in rows if r["answered"]]
+    acc = lambda rs: sum(map(_right, rs)) / len(rs) if rs else None
+    by = {t: acc([r for r in answered if r["type"] == t]) for t in TYPES if any(r["type"] == t for r in rows)}
+    probs = answered and all(r["logprobs"] for r in answered)
+    return Report(n=len(rows), accuracy=acc(answered), by_type=by,
+                  undecided=(1 - len(answered) / len(rows)) if rows else 0.0,
+                  calibration_error=ece(rows) if probs else None)