evalkit-harness 0.1.0__py3-none-any.whl

evalkit/__init__.py

@@ -0,0 +1,5 @@
+"""EvalKit: evaluate your own task across Claude and OpenAI models."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

evalkit/adapters/__init__.py

@@ -0,0 +1,25 @@
+"""Provider adapters. Import providers lazily: the SDKs are optional extras."""
+
+from evalkit.adapters.base import (
+    ALLOWED_PARAMS,
+    BatchRequest,
+    BatchResult,
+    CompletionRequest,
+    ModelAdapter,
+    RawCompletion,
+    discover,
+    get_adapter,
+    register,
+)
+
+__all__ = [
+    "ALLOWED_PARAMS",
+    "BatchRequest",
+    "BatchResult",
+    "CompletionRequest",
+    "ModelAdapter",
+    "RawCompletion",
+    "discover",
+    "get_adapter",
+    "register",
+]

evalkit/adapters/anthropic.py

@@ -0,0 +1,179 @@
+"""Anthropic Messages and Message Batches adapter (ADP-01, ADP-03).
+
+- No sampling parameters (D-22), no refusal fallbacks (D-23), effort set explicitly from
+  request params (D-24), JSON via `output_config.format` (D-24).
+- `output_tokens` includes thinking; thinking is reported separately as tokens_reasoning.
+"""
+
+from __future__ import annotations
+
+import inspect
+import time
+from collections.abc import Mapping
+from typing import Any
+
+from evalkit.adapters.base import (
+    BatchRequest,
+    BatchResult,
+    CompletionRequest,
+    ModelAdapter,
+    RawCompletion,
+    check_params,
+    check_unique_ids,
+    register,
+)
+from evalkit.cost import Price, call_cost, load_prices, price_for
+from evalkit.ratelimit import TokenBucket
+from evalkit.retry import with_retries
+from evalkit.runtime.canonical import to_jsonable
+
+
+@register("anthropic")
+class AnthropicAdapter(ModelAdapter):
+    provider = "anthropic"
+
+    require_effort = True
+
+    def __init__(
+        self,
+        client: Any = None,
+        prices: dict[str, Price] | None = None,
+        bucket: TokenBucket | None = None,
+        **retry: Any,
+    ):
+        self._client = client
+        self._prices = prices if prices is not None else load_prices()
+        self._bucket = bucket  # optional requests-per-second pacing (ADP-07)
+        self._retry = retry
+
+    async def _paced(self, call: Any) -> Any:
+        if self._bucket is not None:
+            await self._bucket.acquire()
+        return await with_retries(call, **self._retry)
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            import anthropic  # optional dependency, imported only when used
+
+            self._client = anthropic.AsyncAnthropic(max_retries=0)
+        return self._client
+
+    def build_params(self, req: CompletionRequest) -> dict[str, Any]:
+        check_params(req, require_effort=self.require_effort)
+        params: dict[str, Any] = {
+            "model": req.model_id,
+            "max_tokens": req.max_output_tokens,
+            "messages": [{"role": "user", "content": req.prompt}],
+        }
+        if req.turns:
+            params["messages"] = [{"role": r, "content": t} for r, t in req.turns]
+        if req.cache:
+            # automatic caching: the breakpoint follows the last block as the history grows
+            params["cache_control"] = {"type": "ephemeral"}
+        if req.system:
+            params["system"] = req.system
+        output_config: dict[str, Any] = {}
+        if "effort" in req.params:
+            output_config["effort"] = req.params["effort"]
+        if req.json_schema is not None:
+            output_config["format"] = {
+                "type": "json_schema",
+                "schema": to_jsonable(req.json_schema),
+            }
+        if output_config:
+            params["output_config"] = output_config
+        return params
+
+    def parse(
+        self, message: Any, req: CompletionRequest, latency_ms: int, *, batch: bool = False
+    ) -> RawCompletion:
+        text = "".join(b.text for b in message.content if getattr(b, "type", "") == "text")
+        u = message.usage
+        cache_read = getattr(u, "cache_read_input_tokens", 0) or 0
+        cache_write = getattr(u, "cache_creation_input_tokens", 0) or 0
+        details = getattr(u, "output_tokens_details", None)
+        thinking = (getattr(details, "thinking_tokens", 0) or 0) if details else 0
+        thinking = min(thinking, u.output_tokens)  # never bill more reasoning than output
+        tokens_in = u.input_tokens + cache_read + cache_write
+        tokens_out = max(u.output_tokens - thinking, 0)
+        cost = call_cost(
+            price_for(self._prices, req.model_id),
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            tokens_reasoning=thinking,
+            tokens_cached=cache_read,
+            tokens_cache_write=cache_write,
+            batch=batch,
+        )
+        stop = getattr(message, "stop_reason", "") or ""
+        return RawCompletion(
+            model=req.model,
+            model_snapshot=getattr(message, "model", "") or "",
+            params=dict(req.params),
+            raw_text=text,
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            tokens_reasoning=thinking,
+            tokens_cached=cache_read,
+            tokens_cache_write=cache_write,
+            latency_ms=latency_ms,
+            cost_usd=cost,
+            request_id=getattr(message, "_request_id", None) or getattr(message, "id", None),
+            stop_reason=stop,
+            refusal=stop == "refusal",
+            batch=batch,
+        )
+
+    async def complete(self, req: CompletionRequest) -> RawCompletion:
+        params = self.build_params(req)
+        t0 = time.perf_counter()
+        message = await self._paced(lambda: self.client.messages.create(**params))
+        return self.parse(message, req, int((time.perf_counter() - t0) * 1000))
+
+    async def submit_batch(self, reqs: list[BatchRequest]) -> str:
+        check_unique_ids(reqs)
+        requests = [
+            {"custom_id": r.custom_id, "params": self.build_params(r.request)} for r in reqs
+        ]
+        batch = await self._paced(lambda: self.client.messages.batches.create(requests=requests))
+        return str(batch.id)
+
+    async def collect_batch(
+        self, batch_id: str, reqs: Mapping[str, CompletionRequest]
+    ) -> list[BatchResult] | None:
+        status = await with_retries(
+            lambda: self.client.messages.batches.retrieve(batch_id), **self._retry
+        )
+        if status.processing_status != "ended":
+            return None
+        results = self.client.messages.batches.results(batch_id)
+        if inspect.isawaitable(results):
+            results = await results
+        out: list[BatchResult] = []
+        seen: set[str] = set()
+        async for item in results:
+            cid = item.custom_id
+            seen.add(cid)
+            if cid not in reqs:
+                continue  # not ours; results arrive in any order and are keyed by custom_id
+            kind = item.result.type
+            if kind == "succeeded":
+                out.append(
+                    BatchResult(
+                        custom_id=cid,
+                        completion=self.parse(
+                            item.result.message, reqs[cid], latency_ms=0, batch=True
+                        ),
+                    )
+                )
+            else:
+                err = getattr(item.result, "error", None)
+                # SDK shape: result.error is an ErrorResponse whose .error carries the type.
+                detail = getattr(getattr(err, "error", None), "type", None) or getattr(
+                    err, "type", None
+                )
+                out.append(BatchResult(custom_id=cid, error=f"{kind}:{detail}" if detail else kind))
+        for cid in sorted(set(reqs) - seen):
+            out.append(BatchResult(custom_id=cid, error="missing_from_results"))
+        return out

evalkit/adapters/base.py

@@ -0,0 +1,176 @@
+"""The adapter interface (ADP-01/02/08/09): every provider looks the same to EvalKit."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import itertools
+import warnings
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Mapping
+from typing import Any, ClassVar, Literal, TypeVar
+
+from pydantic import Field, model_validator
+
+from evalkit.schemas._base import (
+    FrozenMap,
+    JsonValue,
+    Model,
+    Name,
+    NonBlank,
+    NonNegFloat,
+    NonNegInt,
+    PosInt,
+)
+
+ALLOWED_PARAMS = frozenset({"effort"})  # no temperature/top_p/top_k (D-22)
+
+
+class CompletionRequest(Model):
+    model: Name  # alias from config, e.g. "sonnet"
+    model_id: Name  # provider model id, e.g. "claude-sonnet-5-5"
+    prompt: str
+    system: str | None = None
+    max_output_tokens: PosInt = 1024
+    params: FrozenMap[str] = Field(default_factory=dict)
+    json_schema: FrozenMap[JsonValue] | None = None  # structured output (D-24)
+    # Optional multi-turn conversation (an agent's history): alternating user/assistant
+    # turns, starting and ending with a user turn. When set, providers send these turns
+    # instead of `prompt`, and `prompt` must equal `render_turns(turns)` so caching keys see
+    # the same text. Each closed turn is a cacheable prefix: `cache` asks the provider to cache
+    # it (Anthropic automatic caching; OpenAI caches long prefixes on its own). Prefixes inside
+    # one message do not cache (measured, D-51).
+    turns: tuple[tuple[Literal["user", "assistant"], str], ...] | None = None
+    cache: bool = False
+
+    @model_validator(mode="after")
+    def _turns_valid(self) -> CompletionRequest:
+        if self.turns is None:
+            return self
+        roles = [r for r, _ in self.turns]
+        if not roles or roles[0] != "user" or roles[-1] != "user":
+            raise ValueError("turns must start and end with a user turn")
+        if any(a == b for a, b in itertools.pairwise(roles)):
+            raise ValueError("turns must alternate between user and assistant")
+        if render_turns(self.turns) != self.prompt:
+            raise ValueError("prompt must equal render_turns(turns)")
+        return self
+
+
+def render_turns(turns: tuple[tuple[str, str], ...] | list[tuple[str, str]]) -> str:
+    """One text for a conversation (the cache key and providers' fallback)."""
+    return "".join(f"<{role}>\n{text}\n</{role}>\n" for role, text in turns)
+
+
+class RawCompletion(Model):
+    """What a provider returned, before EvalKit parses the answer (A2.1 builds Completion)."""
+
+    model: Name
+    model_snapshot: str
+    params: FrozenMap[str]
+    raw_text: str
+    tokens_in: NonNegInt  # whole prompt, including cached and cache-write tokens
+    tokens_out: NonNegInt  # visible output only
+    tokens_reasoning: NonNegInt = 0  # thinking/reasoning tokens, billed as output
+    tokens_cached: NonNegInt = 0
+    tokens_cache_write: NonNegInt = 0
+    latency_ms: NonNegInt
+    cost_usd: NonNegFloat
+    request_id: str | None = None
+    stop_reason: str = ""
+    refusal: bool = False
+    batch: bool = False
+
+
+class BatchRequest(Model):
+    custom_id: NonBlank
+    request: CompletionRequest
+
+
+class BatchResult(Model):
+    custom_id: NonBlank
+    completion: RawCompletion | None = None
+    error: str | None = None
+
+
+def check_params(req: CompletionRequest, *, require_effort: bool = True) -> None:
+    bad = set(req.params) - ALLOWED_PARAMS
+    if bad:
+        raise ValueError(
+            f"unsupported request params {sorted(bad)}: only {sorted(ALLOWED_PARAMS)} are "
+            "allowed (sampling parameters are rejected by current models, D-22)"
+        )
+    if require_effort and "effort" not in req.params:
+        raise ValueError(
+            f"{req.model}: set params.effort explicitly; per-model defaults differ (D-24)"
+        )
+
+
+def check_unique_ids(reqs: list[BatchRequest]) -> None:
+    ids = [r.custom_id for r in reqs]
+    dupes = sorted({i for i in ids if ids.count(i) > 1})
+    if dupes:
+        raise ValueError(f"duplicate custom_id values in batch: {dupes}")
+
+
+class ModelAdapter(ABC):
+    provider: ClassVar[str]
+
+    @abstractmethod
+    async def complete(self, req: CompletionRequest) -> RawCompletion: ...
+
+    @abstractmethod
+    async def submit_batch(self, reqs: list[BatchRequest]) -> str: ...
+
+    @abstractmethod
+    async def collect_batch(
+        self, batch_id: str, reqs: Mapping[str, CompletionRequest]
+    ) -> list[BatchResult] | None:
+        """None while the batch is still running; otherwise one result per custom_id."""
+
+
+_REGISTRY: dict[str, type[ModelAdapter]] = {}
+ENTRY_POINT_GROUP = "evalkit.adapters"
+_BUILTIN = frozenset({"anthropic", "openai", "openai_compatible"})
+
+
+A = TypeVar("A", bound=type["ModelAdapter"])
+
+
+def register(name: str) -> Callable[[A], A]:
+    def deco(cls: A) -> A:
+        _REGISTRY[name] = cls
+        return cls
+
+    return deco
+
+
+def discover() -> dict[str, type[ModelAdapter]]:
+    """Built-in adapters plus any registered by other packages under `evalkit.adapters`."""
+    from evalkit.adapters import anthropic as _a  # noqa: F401 - registers on import
+    from evalkit.adapters import openai as _o  # noqa: F401
+
+    for ep in importlib.metadata.entry_points(group=ENTRY_POINT_GROUP):
+        if ep.name in _BUILTIN:
+            warnings.warn(f"plugin adapter {ep.name!r} ignored: the name is built in", stacklevel=2)
+            continue
+        if ep.name in _REGISTRY:
+            continue
+        try:
+            cls = ep.load()
+        except Exception as exc:  # a broken plugin must not take down the built-ins
+            warnings.warn(f"plugin adapter {ep.name!r} failed to load: {exc}", stacklevel=2)
+            continue
+        if not (isinstance(cls, type) and issubclass(cls, ModelAdapter)):
+            warnings.warn(
+                f"plugin {ep.name!r} is not a ModelAdapter subclass; ignored", stacklevel=2
+            )
+            continue
+        _REGISTRY[ep.name] = cls
+    return dict(_REGISTRY)
+
+
+def get_adapter(name: str, **kwargs: Any) -> ModelAdapter:
+    adapters = discover()
+    if name not in adapters:
+        raise KeyError(f"unknown adapter {name!r}; available: {sorted(adapters)}")
+    return adapters[name](**kwargs)

evalkit/adapters/openai.py

@@ -0,0 +1,255 @@
+"""OpenAI Chat Completions and Batch adapter (ADP-02, ADP-03), plus OpenAI-compatible
+servers such as Ollama, vLLM or LM Studio (ADP-08).
+
+- Effort maps to `reasoning_effort` (D-24); no sampling parameters (D-22).
+- JSON via `response_format` json_schema (strict). `completion_tokens` includes reasoning;
+  reasoning is reported separately as tokens_reasoning.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections.abc import Mapping
+from typing import Any
+
+from evalkit.adapters.base import (
+    BatchRequest,
+    BatchResult,
+    CompletionRequest,
+    ModelAdapter,
+    RawCompletion,
+    check_params,
+    check_unique_ids,
+    register,
+)
+from evalkit.cost import Price, call_cost, load_prices, price_for
+from evalkit.ratelimit import TokenBucket
+from evalkit.retry import with_retries
+from evalkit.runtime.canonical import to_jsonable
+
+FAILED_BATCH = frozenset({"failed", "expired", "cancelled"})
+
+
+def _get(obj: Any, name: str, default: Any = None) -> Any:
+    if isinstance(obj, Mapping):
+        return obj.get(name, default)
+    return getattr(obj, name, default)
+
+
+@register("openai")
+class OpenAIAdapter(ModelAdapter):
+    provider = "openai"
+    free_if_unknown = False
+
+    require_effort = True
+
+    def __init__(
+        self,
+        client: Any = None,
+        prices: dict[str, Price] | None = None,
+        bucket: TokenBucket | None = None,
+        **retry: Any,
+    ):
+        self._client = client
+        self._prices = prices if prices is not None else load_prices()
+        self._bucket = bucket  # optional requests-per-second pacing (ADP-07)
+        self._retry = retry
+
+    async def _paced(self, call: Any) -> Any:
+        if self._bucket is not None:
+            await self._bucket.acquire()
+        return await with_retries(call, **self._retry)
+
+    def _make_client(self) -> Any:
+        import openai  # optional dependency, imported only when used
+
+        return openai.AsyncOpenAI(max_retries=0)
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            self._client = self._make_client()
+        return self._client
+
+    def build_params(self, req: CompletionRequest) -> dict[str, Any]:
+        check_params(req, require_effort=self.require_effort)
+        messages: list[dict[str, str]] = []
+        if req.system:
+            messages.append({"role": "system", "content": req.system})
+        if req.turns:
+            messages.extend({"role": r, "content": t} for r, t in req.turns)
+        else:
+            messages.append({"role": "user", "content": req.prompt})
+        params: dict[str, Any] = {
+            "model": req.model_id,
+            "messages": messages,
+            "max_completion_tokens": req.max_output_tokens,
+        }
+        if "effort" in req.params:
+            params["reasoning_effort"] = req.params["effort"]
+        if req.json_schema is not None:
+            params["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "answer",
+                    "schema": to_jsonable(req.json_schema),
+                    "strict": True,
+                },
+            }
+        return params
+
+    def parse(
+        self, resp: Any, req: CompletionRequest, latency_ms: int, *, batch: bool = False
+    ) -> RawCompletion:
+        choice = _get(resp, "choices")[0]
+        msg = _get(choice, "message")
+        text = _get(msg, "content") or ""
+        usage = _get(resp, "usage")
+        prompt = _get(usage, "prompt_tokens", 0) or 0
+        completion = _get(usage, "completion_tokens", 0) or 0
+        reasoning = _get(_get(usage, "completion_tokens_details") or {}, "reasoning_tokens", 0) or 0
+        cached = _get(_get(usage, "prompt_tokens_details") or {}, "cached_tokens", 0) or 0
+        tokens_out = max(completion - reasoning, 0)
+        price = price_for(self._prices, req.model_id, free_if_unknown=self.free_if_unknown)
+        cost = call_cost(
+            price,
+            tokens_in=prompt,
+            tokens_out=tokens_out,
+            tokens_reasoning=reasoning,
+            tokens_cached=cached,
+            batch=batch,
+        )
+        refusal = bool(_get(msg, "refusal"))
+        stop = _get(choice, "finish_reason") or ""
+        return RawCompletion(
+            model=req.model,
+            model_snapshot=_get(resp, "model") or "",
+            params=dict(req.params),
+            raw_text=text,
+            tokens_in=prompt,
+            tokens_out=tokens_out,
+            tokens_reasoning=reasoning,
+            tokens_cached=cached,
+            latency_ms=latency_ms,
+            cost_usd=cost,
+            request_id=_get(resp, "_request_id") or _get(resp, "id"),
+            stop_reason="refusal" if refusal else stop,
+            refusal=refusal,
+            batch=batch,
+        )
+
+    async def complete(self, req: CompletionRequest) -> RawCompletion:
+        params = self.build_params(req)
+        t0 = time.perf_counter()
+        resp = await self._paced(lambda: self.client.chat.completions.create(**params))
+        return self.parse(resp, req, int((time.perf_counter() - t0) * 1000))
+
+    async def submit_batch(self, reqs: list[BatchRequest]) -> str:
+        check_unique_ids(reqs)
+        lines = [
+            json.dumps(
+                {
+                    "custom_id": r.custom_id,
+                    "method": "POST",
+                    "url": "/v1/chat/completions",
+                    "body": self.build_params(r.request),
+                },
+                ensure_ascii=False,
+            )
+            for r in reqs
+        ]
+        payload = ("\n".join(lines) + "\n").encode("utf-8")
+        upload = await with_retries(
+            lambda: self.client.files.create(
+                file=("evalkit-batch.jsonl", payload), purpose="batch"
+            ),
+            **self._retry,
+        )
+        batch = await self._paced(
+            lambda: self.client.batches.create(
+                input_file_id=upload.id, endpoint="/v1/chat/completions", completion_window="24h"
+            ),
+        )
+        return str(batch.id)
+
+    async def collect_batch(
+        self, batch_id: str, reqs: Mapping[str, CompletionRequest]
+    ) -> list[BatchResult] | None:
+        batch = await with_retries(lambda: self.client.batches.retrieve(batch_id), **self._retry)
+        if batch.status not in FAILED_BATCH and batch.status != "completed":
+            return None  # validating, in_progress, finalizing, cancelling
+        # Expired or cancelled batches can still hold finished, billed results: read them.
+        missing_error = (
+            "missing_from_results" if batch.status == "completed" else f"batch_{batch.status}"
+        )
+        lines: list[str] = []
+        for file_id in (batch.output_file_id, getattr(batch, "error_file_id", None)):
+            if file_id:
+                content = await with_retries(
+                    lambda f=file_id: self.client.files.content(f), **self._retry
+                )
+                lines += [ln for ln in content.text.splitlines() if ln.strip()]
+        out: list[BatchResult] = []
+        seen: set[str] = set()
+        for line in lines:
+            try:
+                row = json.loads(line)
+            except ValueError:
+                continue  # a damaged line; its request is reported missing below
+            cid = row.get("custom_id")
+            if cid not in reqs or cid in seen:
+                continue
+            seen.add(cid)
+            response = row.get("response") or {}
+            if row.get("error") or response.get("status_code") != 200:
+                err = (row.get("error") or {}).get("code") or f"http_{response.get('status_code')}"
+                out.append(BatchResult(custom_id=cid, error=str(err)))
+            else:
+                out.append(
+                    BatchResult(
+                        custom_id=cid,
+                        completion=self.parse(
+                            response["body"], reqs[cid], latency_ms=0, batch=True
+                        ),
+                    )
+                )
+        for cid in sorted(set(reqs) - seen):
+            out.append(BatchResult(custom_id=cid, error=missing_error))
+        return out
+
+
+@register("openai_compatible")
+class OpenAICompatibleAdapter(OpenAIAdapter):
+    """Any server that speaks the OpenAI Chat Completions API (ADP-08). Models missing from
+    prices.yaml are costed at $0 with a warning, since local models are usually free."""
+
+    provider = "openai_compatible"
+    free_if_unknown = True
+    require_effort = False  # local models often have no reasoning effort setting
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key_env: str | None = None,
+        client: Any = None,
+        prices: dict[str, Price] | None = None,
+        **retry: Any,
+    ):
+        if not base_url.startswith(("http://", "https://")):
+            raise ValueError(f"base_url must start with http:// or https://, got {base_url!r}")
+        super().__init__(client=client, prices=prices, **retry)
+        self.base_url = base_url
+        self.api_key_env = api_key_env
+
+    def _make_client(self) -> Any:
+        import openai
+
+        if self.api_key_env:
+            key = os.environ.get(self.api_key_env)
+            if not key:
+                raise ValueError(f"environment variable {self.api_key_env} is not set")
+        else:
+            key = "not-needed"  # servers such as Ollama ignore the key
+        return openai.AsyncOpenAI(base_url=self.base_url, api_key=key, max_retries=0)