jevify 0.1.0__py3-none-any.whl

jevify/adapters/endpoint/adapter.py

@@ -0,0 +1,281 @@
+"""EndpointBackend: the backend port over an OpenAI-compatible server."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import math
+import time
+from collections.abc import Callable, Sequence
+
+from jevify.adapters.endpoint.client import OpenAICompatibleClient
+from jevify.adapters.endpoint.dialects import base_request, parse_response
+from jevify.adapters.endpoint.readout import RUNGS, MissingLabelsError
+from jevify.domain.distribution import Distribution
+from jevify.domain.questions import Question, State
+from jevify.ports.backend import (
+    RUNG_RANK,
+    BackendError,
+    Capabilities,
+    Dialect,
+    RawAnswer,
+    Rung,
+    StateHandle,
+)
+from jevify.recipes.render import render_prefix, render_question
+from jevify.recipes.schema import Recipe
+
+Clock = Callable[[], float]
+
+
+class EndpointBackend:
+    def __init__(
+        self,
+        client: OpenAICompatibleClient,
+        recipe: Recipe,
+        *,
+        clock: Clock = time.perf_counter,
+        concurrency: int = 1,
+    ) -> None:
+        if recipe.endpoint is None:
+            raise BackendError("an endpoint backend needs a recipe with an endpoint section")
+        self.client = client
+        self.recipe = recipe
+        self.clock = clock
+        self.dialect = (
+            Dialect(recipe.endpoint.dialect)
+            if recipe.endpoint.dialect != "auto"
+            else Dialect.GENERIC
+        )
+        self.capabilities = Capabilities.from_dict(recipe.probe) if recipe.probe else None
+        probed = (
+            self.capabilities.fanout.concurrency
+            if self.capabilities and self.capabilities.fanout
+            else 1
+        )
+        self.concurrency = max(1, concurrency, probed)
+        self.slots = self.capabilities.slots if self.capabilities else None
+
+    async def probe(self) -> Capabilities:
+        from jevify.adapters.endpoint.probe import Prober
+
+        return await Prober(self.client, self.recipe, self.clock).run()
+
+    async def warm(self, state: State) -> StateHandle:
+        """Send the state-only prefix once (one token, nothing read) so the server's cache
+        holds it; on llama.cpp once per slot the fan-out will use."""
+        wanted = state.modalities - {"text"}
+        if wanted and self.capabilities and not wanted <= self.capabilities.modalities:
+            missing = sorted(wanted - self.capabilities.modalities)
+            raise BackendError(
+                f"this backend was probed without {missing} support; the state carries "
+                f"{missing} parts and jevify never drops them silently (INV-9)"
+            )
+        prefix = render_prefix(self.recipe, state)
+        identity = json.dumps(
+            [prefix.mode, prefix.system, prefix.text, prefix.kwargs, len(prefix.images)],
+            sort_keys=True,
+            ensure_ascii=False,
+        )
+        state_id = hashlib.sha256(identity.encode("utf-8")).hexdigest()
+        slots = tuple(range(min(self.concurrency, self.slots))) if self.slots else ()
+        started = self.clock()
+        cached: int | None = None
+        for slot in slots or (None,):
+            path, body = base_request(self.recipe.model.name, prefix, "", self.dialect)
+            if slot is not None:
+                body["id_slot"] = slot
+            post = self.client.post_root if path == "completion" else self.client.post_v1
+            parsed = parse_response(path, await post(path, body), self.dialect)
+            cached = parsed.cached_tokens
+        warm_ms = (self.clock() - started) * 1000.0
+        return StateHandle(
+            state_id=state_id, prefix=prefix, warm_ms=warm_ms, cached_tokens=cached, slots=slots
+        )
+
+    async def evaluate(self, handle: StateHandle, questions: Sequence[Question]) -> list[RawAnswer]:
+        ladder = self._ladder()
+        gate = asyncio.Semaphore(self.concurrency)
+
+        async def one(index: int, question: Question) -> RawAnswer:
+            slot = handle.slots[index % len(handle.slots)] if handle.slots else None
+            async with gate:
+                for step, rung in enumerate(ladder):
+                    try:
+                        return await self._answer(handle, question, rung, slot)
+                    except MissingLabelsError:
+                        if step == len(ladder) - 1:
+                            raise
+                raise AssertionError("unreachable: the ladder is never empty")
+
+        return list(await asyncio.gather(*(one(i, q) for i, q in enumerate(questions))))
+
+    def _ladder(self) -> list[Rung]:
+        """The rungs a question may be read on, strongest first. A pinned rung stands alone
+        and fails loudly; auto walks every proven rung down to the floor."""
+        strongest = self._resolve_rung()
+        if self.recipe.readout.rung != "auto":
+            return [strongest]
+        proven = Capabilities.from_dict(self.recipe.probe).rungs if self.recipe.probe else set()
+        return [r for r in RUNG_RANK if r in proven and r in RUNGS]
+
+    def _resolve_rung(self) -> Rung:
+        pinned = self.recipe.readout.rung
+        proven = (
+            Capabilities.from_dict(self.recipe.probe).rungs if self.recipe.probe else frozenset()
+        )
+        if pinned == "auto":
+            if not self.recipe.probe:
+                raise BackendError(
+                    "readout.rung is auto but the recipe has no probe section; "
+                    "run `jevify probe <recipe>` or pin a rung"
+                )
+            for rung in RUNG_RANK:
+                if rung in proven and rung in RUNGS:
+                    return rung
+            raise BackendError("the probe proved no rung this build implements")
+        rung = Rung(pinned)
+        if rung not in RUNGS:
+            raise BackendError(f"rung {rung} is not implemented yet")
+        if self.recipe.probe and rung not in proven:
+            raise BackendError(
+                f"readout.rung {rung} is pinned but the probe did not prove it; "
+                f"proven: {[str(r) for r in RUNG_RANK if r in proven]}"
+            )
+        return rung
+
+    def _orders(self, question: Question, rendered) -> list[tuple[int, ...]] | None:
+        """Cyclic option orders when the recipe asks for permutation on an identifier-based
+        choice or score question; None means a single order."""
+        policy = self.recipe.readout.permutation
+        n = len(question.labels)
+        if policy == "none" or rendered.composition != "single" or n < 2:
+            return None
+        if policy == "full":
+            import itertools
+
+            return list(itertools.permutations(range(n)))
+        k = min(self.recipe.readout.permutation_calls, n)
+        return [tuple((i + shift) % n for i in range(n)) for shift in range(k)]
+
+    async def _permuted(
+        self,
+        handle: StateHandle,
+        question: Question,
+        rung: Rung,
+        slot: int | None,
+        orders: list[tuple[int, ...]],
+    ) -> RawAnswer:
+        """Ask once per order and average the normalized distributions, so position bias
+        cancels; the record says how many calls it took."""
+        impl = RUNGS[rung]
+        sums: dict[str, float] = dict.fromkeys(question.labels, 0.0)
+        latency_ms = 0.0
+        prompt_tokens = 0
+        cached_tokens: int | None = None
+        degraded = False
+        off_menu: list[float] = []
+        raw_parts = []
+        for order in orders:
+            part = render_question(self.recipe, question, order=order).parts[0]
+            path, body = base_request(
+                self.recipe.model.name, handle.prefix, part.text, self.dialect
+            )
+            impl.shape(body, part, self.recipe.readout.top_k, self.dialect)
+            if slot is not None:
+                body["id_slot"] = slot
+            started = self.clock()
+            post = self.client.post_root if path == "completion" else self.client.post_v1
+            response = await post(path, body)
+            latency_ms += (self.clock() - started) * 1000.0
+            parsed = parse_response(path, response, self.dialect)
+            readout = impl.read(parsed, part)
+            distribution = Distribution.from_logprobs(readout.logprobs)
+            for label, probability in distribution.as_mapping().items():
+                sums[label] += probability
+            degraded = degraded or readout.degraded
+            if readout.off_menu_mass is not None:
+                off_menu.append(readout.off_menu_mass)
+            prompt_tokens += parsed.prompt_tokens or 0
+            cached_tokens = parsed.cached_tokens
+            raw_parts.append(
+                {
+                    "path": path,
+                    "order": list(order),
+                    "entries": [e.__dict__ for e in parsed.entries],
+                }
+            )
+        k = len(orders)
+        logprobs = {label: math.log(max(total / k, 1e-300)) for label, total in sums.items()}
+        return RawAnswer(
+            question_id=question.id,
+            logprobs=logprobs,
+            semantics="readout",
+            rung=rung,
+            degraded=degraded,
+            off_menu_mass=(sum(off_menu) / len(off_menu)) if off_menu else None,
+            latency_ms=latency_ms,
+            prompt_tokens=prompt_tokens,
+            cached_tokens=cached_tokens,
+            calls=k,
+            composition="permuted",
+            raw={"parts": raw_parts},
+        )
+
+    async def _answer(
+        self, handle: StateHandle, question: Question, rung: Rung, slot: int | None = None
+    ) -> RawAnswer:
+        """One request per part; a single-part question is the readout itself, a per-option
+        question composes each option's yes-minus-no logit into one distribution."""
+        rendered = render_question(self.recipe, question)
+        orders = self._orders(question, rendered)
+        if orders is not None:
+            return await self._permuted(handle, question, rung, slot, orders)
+        impl = RUNGS[rung]
+        readouts = []
+        latency_ms = 0.0
+        prompt_tokens = 0
+        cached_tokens: int | None = None
+        raw_parts = []
+        for part in rendered.parts:
+            path, body = base_request(
+                self.recipe.model.name, handle.prefix, part.text, self.dialect
+            )
+            impl.shape(body, part, self.recipe.readout.top_k, self.dialect)
+            if slot is not None:
+                body["id_slot"] = slot
+            started = self.clock()
+            post = self.client.post_root if path == "completion" else self.client.post_v1
+            response = await post(path, body)
+            latency_ms += (self.clock() - started) * 1000.0
+            parsed = parse_response(path, response, self.dialect)
+            readouts.append(impl.read(parsed, part))
+            prompt_tokens += parsed.prompt_tokens or 0
+            cached_tokens = parsed.cached_tokens
+            raw_parts.append({"path": path, "entries": [e.__dict__ for e in parsed.entries]})
+        if rendered.composition == "single":
+            readout = readouts[0]
+            logprobs, off_menu, missing = readout.logprobs, readout.off_menu_mass, readout.missing
+        else:
+            logprobs = {
+                part.label: r.logprobs["true"] - r.logprobs["false"]
+                for part, r in zip(rendered.parts, readouts, strict=True)
+                if part.label is not None
+            }
+            off_menu, missing = None, ()
+        return RawAnswer(
+            question_id=question.id,
+            logprobs=logprobs,
+            semantics="readout",
+            rung=rung,
+            degraded=any(r.degraded for r in readouts),
+            off_menu_mass=off_menu,
+            latency_ms=latency_ms,
+            prompt_tokens=prompt_tokens,
+            cached_tokens=cached_tokens,
+            missing=missing,
+            calls=len(rendered.parts),
+            composition=rendered.composition,
+            raw={"parts": raw_parts},
+        )

jevify/adapters/endpoint/client.py

@@ -0,0 +1,114 @@
+"""The only module that imports httpx. Timeouts, bounded retries, error wrapping."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import re
+from typing import Any
+
+import httpx
+
+from jevify.ports.backend import BackendError, ContextLimitError
+
+_RETRY_STATUSES = {429, 500, 502, 503, 504}
+_CONTEXT_LIMIT = re.compile(
+    r"(maximum context length|context length|too many tokens|exceeds)", re.I
+)
+
+
+class OpenAICompatibleClient:
+    """HTTP access to one server. `v1/*` for the OpenAI surface, root for server extras."""
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        api_key: str | None = None,
+        timeout_s: float = 120.0,
+        http: httpx.AsyncClient | None = None,
+        max_retries: int = 3,
+    ) -> None:
+        stripped = base_url.rstrip("/")
+        self.root = stripped[: -len("/v1")] if stripped.endswith("/v1") else stripped
+        self.v1 = self.root + "/v1"
+        self.api_key = api_key
+        self.timeout_s = timeout_s
+        self.max_retries = max_retries
+        self._http = http or httpx.AsyncClient(timeout=httpx.Timeout(timeout_s))
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    async def _request(self, method: str, url: str, body: dict[str, Any] | None = None) -> Any:
+        attempt = 0
+        while True:
+            try:
+                response = await self._http.request(
+                    method, url, json=body, headers=self._headers(), timeout=self.timeout_s
+                )
+            except httpx.HTTPError as exc:
+                raise BackendError(f"{method} {url} failed: {exc}") from exc
+            if response.status_code in _RETRY_STATUSES and attempt < self.max_retries:
+                attempt += 1
+                await asyncio.sleep(_retry_delay(response, attempt))
+                continue
+            if response.status_code >= 400:
+                raise _http_error(method, url, response)
+            return response.json()
+
+    async def post_v1(self, path: str, body: dict[str, Any]) -> Any:
+        return await self._request("POST", f"{self.v1}/{path.lstrip('/')}", body)
+
+    async def get_v1(self, path: str) -> Any:
+        return await self._request("GET", f"{self.v1}/{path.lstrip('/')}")
+
+    async def post_root(self, path: str, body: dict[str, Any]) -> Any:
+        return await self._request("POST", f"{self.root}/{path.lstrip('/')}", body)
+
+    async def get_root(self, path: str) -> Any:
+        return await self._request("GET", f"{self.root}/{path.lstrip('/')}")
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+
+def _retry_delay(response: httpx.Response, attempt: int) -> float:
+    header = response.headers.get("retry-after-ms") or response.headers.get("retry-after")
+    if header:
+        try:
+            value = float(header)
+            return (
+                value / 1000.0 if "ms" in (response.headers.get("retry-after-ms") or "") else value
+            )
+        except ValueError:
+            pass
+    return min(2.0, 0.05 * (2**attempt)) + random.uniform(0.0, 0.05)  # noqa: S311 - jitter only
+
+
+def _http_error(method: str, url: str, response: httpx.Response) -> BackendError:
+    message = _error_message(response)
+    text = f"{method} {url} returned {response.status_code}: {message}"
+    if response.status_code in (400, 413, 422) and _CONTEXT_LIMIT.search(message):
+        return ContextLimitError(text, status=response.status_code)
+    return BackendError(text, status=response.status_code)
+
+
+def _error_message(response: httpx.Response) -> str:
+    try:
+        body = response.json()
+    except ValueError:
+        return response.text[:500]
+    if isinstance(body, dict):
+        error = body.get("error")
+        if isinstance(error, dict) and "message" in error:
+            return str(error["message"])
+        if isinstance(error, str):
+            return error
+        for key in ("message", "detail"):
+            if key in body:
+                return str(body[key])
+    return str(body)[:500]

jevify/adapters/endpoint/dialects.py

@@ -0,0 +1,166 @@
+"""The dialect registry: how each server spells the OpenAI surface and its extras.
+
+The only file where the dialect names appear as behaviour. The adapter asks a
+dialect to shape a request and to parse a response; it never branches on the
+name itself.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from jevify.ports.backend import Dialect
+from jevify.recipes.render import RenderedPrefix
+
+
+@dataclass(frozen=True)
+class Entry:
+    """One token at the answer position: decoded text, id when the server gives one, logprob."""
+
+    text: str
+    logprob: float
+    id: int | None = None
+
+
+@dataclass(frozen=True)
+class Parsed:
+    entries: tuple[Entry, ...]
+    prompt_tokens: int | None
+    cached_tokens: int | None
+    completion_tokens: int | None
+
+
+def base_request(
+    model: str, prefix: RenderedPrefix, suffix: str, dialect: Dialect
+) -> tuple[str, dict[str, Any]]:
+    """The readout request every rung starts from: one token, no sampling, logprobs on.
+
+    Returns the route key and the body: "chat/completions" and "completions" live under
+    /v1; "completion" is llama.cpp's native root route, used for raw multimodal input.
+    """
+    if prefix.mode == "raw":
+        if prefix.images:
+            if dialect is not Dialect.LLAMACPP:
+                raise ValueError("raw-mode images need the llama.cpp native completion route")
+            # llama.cpp's documented shape: a prompt object with the text (one media marker
+            # per image, the marker the server reports in /props) and the base64 payloads.
+            # A top-level multimodal_data key is accepted and silently ignored: the image
+            # never reaches the model. The probe checks that an image adds prompt tokens.
+            return "completion", {
+                "prompt": {
+                    "prompt_string": prefix.text + suffix,
+                    "multimodal_data": [_base64_payload(i.data_uri) for i in prefix.images],
+                },
+                "n_predict": 1,
+                "temperature": 0,
+                "stream": False,
+            }
+        body: dict[str, Any] = {
+            "model": model,
+            "prompt": prefix.text + suffix,
+            "max_tokens": 1,
+            "temperature": 0,
+            "stream": False,
+        }
+        return "completions", body
+    content: str | list[dict[str, Any]]
+    if prefix.images:
+        content = [{"type": "text", "text": prefix.text}]
+        content.extend(
+            {"type": "image_url", "image_url": {"url": image.data_uri}} for image in prefix.images
+        )
+        content.append({"type": "text", "text": suffix})
+    else:
+        content = prefix.text + suffix
+    messages: list[dict[str, Any]] = []
+    if prefix.system:
+        messages.append({"role": "system", "content": prefix.system})
+    messages.append({"role": "user", "content": content})
+    body = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": 1,
+        "temperature": 0,
+        "stream": False,
+        "logprobs": True,
+    }
+    if prefix.prefill:
+        messages.append({"role": "assistant", "content": prefix.prefill})
+        body["continue_final_message"] = True
+        body["add_generation_prompt"] = False
+    if prefix.kwargs and dialect is not Dialect.GENERIC:
+        body["chat_template_kwargs"] = dict(prefix.kwargs)
+    return "chat/completions", body
+
+
+def _base64_payload(data_uri: str) -> str:
+    return data_uri.split(",", 1)[1] if data_uri.startswith("data:") else data_uri
+
+
+def neutral_samplers(body: dict[str, Any]) -> None:
+    """llama.cpp only: post-sampling probabilities are read after the sampler chain, so the
+    chain must not truncate or reshape the distribution (the server's defaults do)."""
+    body.update({"temperature": 1.0, "top_k": 0, "top_p": 1.0, "min_p": 0.0})
+    body["post_sampling_probs"] = True
+
+
+def grammar_for(texts: list[str]) -> str:
+    """A GBNF root that admits exactly the answer spellings."""
+    escaped = " | ".join('"' + t.replace("\\", "\\\\").replace('"', '\\"') + '"' for t in texts)
+    return f"root ::= {escaped}"
+
+
+def parse_response(path: str, response: dict[str, Any], dialect: Dialect) -> Parsed:
+    entries: list[Entry] = []
+    if path == "completion":
+        # llama.cpp native route: completion_probabilities plus top-level counters.
+        items = response.get("completion_probabilities") or []
+        if items:
+            for item in items[0].get("top_logprobs") or items[0].get("top_probs") or []:
+                entries.append(_entry_from(item, dialect))
+        return Parsed(
+            entries=tuple(entries),
+            prompt_tokens=response.get("tokens_evaluated"),
+            cached_tokens=response.get("tokens_cached"),
+            completion_tokens=response.get("tokens_predicted"),
+        )
+    choice = response["choices"][0]
+    logprobs = choice.get("logprobs") or {}
+    # Shape-driven, not dialect-driven: llama.cpp answers the completions route with the
+    # chat-style `content` list (ids included); OpenAI and vLLM use the dict form there.
+    if "content" in logprobs:
+        content = logprobs.get("content") or []
+        if content:
+            for item in content[0].get("top_logprobs") or content[0].get("top_probs") or []:
+                entries.append(_entry_from(item, dialect))
+    else:
+        top = (logprobs.get("top_logprobs") or [{}])[0] or {}
+        entries = [Entry(text=text, logprob=float(lp)) for text, lp in top.items()]
+    usage = response.get("usage") or {}
+    details = usage.get("prompt_tokens_details") or {}
+    cached = details.get("cached_tokens")
+    if cached is None and "tokens_cached" in response:
+        cached = response.get("tokens_cached")
+    return Parsed(
+        entries=tuple(entries),
+        prompt_tokens=usage.get("prompt_tokens"),
+        cached_tokens=cached,
+        completion_tokens=usage.get("completion_tokens"),
+    )
+
+
+def _entry_from(item: dict[str, Any], dialect: Dialect) -> Entry:
+    text = str(item.get("token", ""))
+    ident = item.get("id")
+    if ident is None and text.startswith("token_id:"):
+        try:
+            ident = int(text[len("token_id:") :])
+        except ValueError:
+            ident = None
+    value = item.get("logprob")
+    if value is None and "prob" in item:
+        import math
+
+        value = math.log(item["prob"]) if item["prob"] > 0 else -1e4
+    return Entry(text=text, logprob=float(value), id=ident)