evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/client/api.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+from httpx_retries import Retry, RetryTransport
+
+from evalkit import version
+
+RETRY_STATUSES = (502, 503, 504)
+RETRY_ERRORS = (httpx.NetworkError, httpx.ConnectTimeout, httpx.RemoteProtocolError)
+RETRY_ATTEMPTS = 4
+RETRY_BACKOFF_FACTOR = 0.5
+RETRY_MAX_WAIT_SECONDS = 8.0
+READ_METHODS = ("GET", "HEAD")
+WRITE_METHODS = ("GET", "HEAD", "POST", "PUT", "PATCH")
+
+
+class ApiRequestError(Exception):
+    def __init__(self, status: int, detail: str, method: str, path: str):
+        self.status = status
+        self.detail = detail
+        super().__init__(f"{method} {path} → {status}: {detail}")
+
+    @classmethod
+    def from_response(cls, resp: httpx.Response) -> ApiRequestError:
+        return cls(
+            resp.status_code, response_detail(resp), resp.request.method, resp.request.url.path
+        )
+
+
+def response_detail(resp: httpx.Response) -> str:
+    if resp.content and resp.headers.get("content-type", "").startswith("application/json"):
+        body = resp.json()
+        if isinstance(body, dict) and body.get("detail") is not None:
+            return str(body["detail"])
+    return resp.reason_phrase
+
+
+def retry_policy(methods: tuple[str, ...]) -> Retry:
+    return Retry(
+        total=RETRY_ATTEMPTS,
+        allowed_methods=methods,
+        status_forcelist=RETRY_STATUSES,
+        retry_on_exceptions=RETRY_ERRORS,
+        backoff_factor=RETRY_BACKOFF_FACTOR,
+        max_backoff_wait=RETRY_MAX_WAIT_SECONDS,
+    )
+
+
+def retry_extensions(retry: bool | None) -> dict[str, Retry]:
+    if retry is None:
+        return {}
+    return {"retry": retry_policy(WRITE_METHODS) if retry else Retry(total=0)}
+
+
+class VersionMismatch(Exception):
+    def __init__(self, local: str, remote: str):
+        self.local = local
+        self.remote = remote
+        super().__init__(
+            f"evalkit {local} does not match server {remote}; local runs need the same "
+            "version. Run `evalkit update` from the project directory: it reinstalls the CLI "
+            "and moves the evalkit-py pin in ./pyproject.toml"
+        )
+
+
+# Shorter than uvicorn's 5s idle keep-alive, so a poll never reuses a
+# connection the server is closing at the same moment.
+KEEPALIVE_EXPIRY_SECONDS = 2
+DEFAULT_TIMEOUT_SECONDS = 120
+
+
+def timeout_seconds() -> float:
+    return float(os.environ.get("EVALKIT_TIMEOUT", DEFAULT_TIMEOUT_SECONDS))
+
+
+def encode(segment: str) -> str:
+    return quote(segment, safe="")
+
+
+class AsyncApi:
+    def __init__(
+        self,
+        server_url: str,
+        token: str | None,
+        *,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ):
+        self._client = httpx.AsyncClient(
+            base_url=server_url.rstrip("/"),
+            headers={"Authorization": f"Bearer {token}"} if token else {},
+            timeout=timeout_seconds(),
+            transport=RetryTransport(
+                transport=transport
+                or httpx.AsyncHTTPTransport(
+                    limits=httpx.Limits(keepalive_expiry=KEEPALIVE_EXPIRY_SECONDS)
+                ),
+                retry=retry_policy(READ_METHODS),
+            ),
+        )
+        self.remote_version: str | None = None
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def request(
+        self, method: str, path: str, *, retry: bool | None = None, **kwargs
+    ) -> httpx.Response:
+        resp = await self._client.request(
+            method, path, extensions=retry_extensions(retry), **kwargs
+        )
+        self.remote_version = resp.headers.get("X-Evalkit-Version", self.remote_version)
+        if resp.status_code >= 400:
+            raise ApiRequestError.from_response(resp)
+        return resp
+
+    async def get(self, path: str, **kwargs) -> httpx.Response:
+        return await self.request("GET", path, **kwargs)
+
+    async def post(self, path: str, **kwargs) -> httpx.Response:
+        return await self.request("POST", path, **kwargs)
+
+    async def put(self, path: str, **kwargs) -> httpx.Response:
+        return await self.request("PUT", path, **kwargs)
+
+    async def patch(self, path: str, **kwargs) -> httpx.Response:
+        return await self.request("PATCH", path, **kwargs)
+
+    async def head(self, path: str, **kwargs) -> httpx.Response:
+        resp = await self._client.request("HEAD", path, **kwargs)
+        self.remote_version = resp.headers.get("X-Evalkit-Version", self.remote_version)
+        return resp
+
+    async def require_same_version(self) -> None:
+        await self.get("/api/health")
+        local = version.current()
+        if self.remote_version != local:
+            raise VersionMismatch(local, self.remote_version or "unknown")
+
+
+class SyncApi:
+    def __init__(
+        self,
+        server_url: str,
+        token: str | None = None,
+        *,
+        transport: httpx.BaseTransport | None = None,
+    ):
+        self._client = httpx.Client(
+            base_url=server_url.rstrip("/"),
+            headers={"Authorization": f"Bearer {token}"} if token else {},
+            timeout=timeout_seconds(),
+            transport=RetryTransport(
+                transport=transport or httpx.HTTPTransport(), retry=retry_policy(READ_METHODS)
+            ),
+        )
+
+    def _handle(self, resp: httpx.Response) -> httpx.Response:
+        if resp.status_code >= 400:
+            raise ApiRequestError.from_response(resp)
+        return resp
+
+    def request(
+        self, method: str, path: str, *, retry: bool | None = None, **kwargs
+    ) -> httpx.Response:
+        return self._handle(
+            self._client.request(method, path, extensions=retry_extensions(retry), **kwargs)
+        )
+
+    def get(self, path: str, **kwargs) -> httpx.Response:
+        return self.request("GET", path, **kwargs)
+
+    def post(self, path: str, **kwargs) -> httpx.Response:
+        return self.request("POST", path, **kwargs)
+
+    def delete(self, path: str, **kwargs) -> httpx.Response:
+        return self.request("DELETE", path, **kwargs)
+
+    def post_files(self, path: str, files: list[tuple[str, Path]], **kwargs) -> httpx.Response:
+        parts = [("files", (name, f.read_bytes(), "application/octet-stream")) for name, f in files]
+        return self.request("POST", path, files=parts, **kwargs)
+
+    def patch(self, path: str, **kwargs) -> httpx.Response:
+        return self.request("PATCH", path, **kwargs)
+
+    def put(self, path: str, **kwargs) -> httpx.Response:
+        return self.request("PUT", path, **kwargs)
+
+    def head(self, path: str, **kwargs) -> httpx.Response:
+        return self._client.request("HEAD", path, **kwargs)

evalkit/client/blob.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from pathlib import Path
+from tempfile import mkdtemp
+
+from evalkit.client.api import AsyncApi
+from evalkit.store.base import BlobStream, CaseFile
+
+FIXTURE_PREFIX = "fixtures/sha256/"
+CHECK_PREFIX = "checks/"
+
+
+class HttpBlobStore:
+    def __init__(self, api: AsyncApi, experiment_id: int | None, cache_dir: Path | None = None):
+        self._api = api
+        self._experiment_id = experiment_id
+        self._configured_dir = cache_dir
+        self._scratch_dir: Path | None = None
+
+    def _cache_dir(self) -> Path:
+        if self._configured_dir is not None:
+            return self._configured_dir
+        if self._scratch_dir is None:
+            self._scratch_dir = Path(mkdtemp(prefix="evalkit-fixtures-"))
+        return self._scratch_dir
+
+    def _local_path(self, key: str) -> Path | None:
+        if key.startswith(FIXTURE_PREFIX):
+            return self._cache_dir() / ".sha256" / key.removeprefix(FIXTURE_PREFIX)
+        if key.startswith(CHECK_PREFIX):
+            return self._cache_dir() / ".checks" / key.removeprefix(CHECK_PREFIX)
+        return None
+
+    def _experiment_path(self) -> str:
+        if self._experiment_id is None:
+            raise ValueError("no experiment bound; only fixture and check keys are available")
+        return f"/api/experiments/{self._experiment_id}/blob"
+
+    def _remote(self, key: str) -> str:
+        return f"{self._experiment_path()}/{key}"
+
+    async def put_object(self, key: str, data: bytes) -> None:
+        local = self._local_path(key)
+        if local is not None:
+            local.parent.mkdir(parents=True, exist_ok=True)
+            local.write_bytes(data)
+            return
+        await self._api.put(self._remote(key), content=data, retry=True)
+
+    async def put_stream(self, key: str, chunks: AsyncIterator[bytes]) -> None:
+        await self.put_object(key, b"".join([chunk async for chunk in chunks]))
+
+    async def _cached(self, key: str, local: Path) -> Path:
+        if not local.is_file() and key.startswith(FIXTURE_PREFIX):
+            resp = await self._api.get(f"/api/fixtures/sha256/{key.removeprefix(FIXTURE_PREFIX)}")
+            local.parent.mkdir(parents=True, exist_ok=True)
+            local.write_bytes(resp.content)
+        return local
+
+    async def get_object(self, key: str) -> bytes:
+        local = self._local_path(key)
+        if local is None:
+            return (await self._api.get(self._remote(key))).content
+        return (await self._cached(key, local)).read_bytes()
+
+    async def get_stream(self, key: str) -> BlobStream:
+        data = await self.get_object(key)
+        return BlobStream(len(data), _once(data))
+
+    async def fixture_path(self, file: CaseFile) -> Path:
+        named = self._cache_dir() / file.sha256 / file.key
+        if not named.is_file():
+            named.parent.mkdir(parents=True, exist_ok=True)
+            named.write_bytes(await self.get_object(f"{FIXTURE_PREFIX}{file.sha256}"))
+        return named
+
+    async def object_exists(self, key: str) -> bool:
+        local = self._local_path(key)
+        if local is not None:
+            return local.is_file()
+        return (await self._api.head(self._remote(key))).status_code == 200
+
+    async def list_objects(self, prefix: str) -> list[tuple[str, int]]:
+        if self._local_path(prefix) is not None:
+            raise ValueError("listing local keys is not supported")
+        body = (await self._api.get(self._experiment_path(), params={"prefix": prefix})).json()
+        return [(str(key), int(size)) for key, size in body["objects"]]
+
+    async def delete_object(self, key: str) -> None:
+        raise NotImplementedError
+
+    async def copy_object(self, source: str, destination: str) -> None:
+        raise NotImplementedError
+
+    async def object_url(self, key: str) -> str:
+        local = self._local_path(key)
+        if local is None:
+            raise ValueError(f"{key} has no URL a harness can fetch")
+        return (await self._cached(key, local)).resolve().as_uri()
+
+
+async def _once(data: bytes) -> AsyncIterator[bytes]:
+    yield data

evalkit/client/connection.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import tomllib
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import NamedTuple
+
+CONFIG_PATH = Path.home() / ".evalkit" / "config.toml"
+
+TOKEN_WARNING_WINDOW = timedelta(days=1)
+
+
+class Connection(NamedTuple):
+    url: str
+    token: str | None
+    username: str | None
+    from_env: bool = False
+
+
+def _toml_string(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def load(path: Path | None = None) -> Connection | None:
+    target = path or CONFIG_PATH
+    if not target.is_file():
+        return None
+    data = tomllib.loads(target.read_text())
+    if "url" not in data:
+        if "profiles" in data:
+            raise LookupError(
+                f"{target} holds server profiles, which evalkit no longer supports; "
+                "run `evalkit login --url <url>` to replace it"
+            )
+        return None
+    return Connection(str(data["url"]), data.get("token"), data.get("username"))
+
+
+def save(connection: Connection, path: Path | None = None) -> Path:
+    target = path or CONFIG_PATH
+    lines = [f"url = {_toml_string(connection.url)}"]
+    if connection.token is not None:
+        lines.append(f"token = {_toml_string(connection.token)}")
+    if connection.username is not None:
+        lines.append(f"username = {_toml_string(connection.username)}")
+    target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    target.parent.chmod(0o700)
+    fd, temp = tempfile.mkstemp(dir=target.parent, prefix=f".{target.name}.")
+    with os.fdopen(fd, "w") as file:
+        file.write("\n".join(lines) + "\n")
+    os.replace(temp, target)
+    return target
+
+
+def active(path: Path | None = None) -> Connection:
+    env_url = os.environ.get("EVALKIT_SERVER_URL")
+    if env_url:
+        return Connection(env_url, os.environ.get("EVALKIT_TOKEN"), None, from_env=True)
+    stored = load(path or CONFIG_PATH)
+    if stored is None:
+        raise LookupError(
+            "no server configured — run `evalkit login --url <url>` or set EVALKIT_SERVER_URL"
+        )
+    return stored
+
+
+def renew_hint(connection: Connection) -> str:
+    if connection.from_env:
+        return "set a fresh EVALKIT_TOKEN"
+    return f"run `evalkit login --url {connection.url}`"
+
+
+def token_expiry_warning(
+    connection: Connection, expires_at: str | None, now: datetime
+) -> str | None:
+    if expires_at is None:
+        return None
+    left = datetime.fromisoformat(expires_at) - now
+    if left > TOKEN_WARNING_WINDOW:
+        return None
+    hours = max(left.total_seconds(), 0) / 3600
+    return (
+        f"the token for {connection.url} expires in {hours:.1f}h; "
+        f"{renew_hint(connection)} before a long run"
+    )

evalkit/client/evaluators.py

@@ -0,0 +1,11 @@
+from evalkit.client.api import AsyncApi
+from evalkit.core.remote_evaluators import RemoteEvaluator
+
+
+class HttpEvaluators:
+    def __init__(self, api: AsyncApi):
+        self._api = api
+
+    async def in_workspace(self, workspace: str) -> dict[str, RemoteEvaluator]:
+        rows = (await self._api.get("/api/evaluators", params={"workspace": workspace})).json()
+        return {row["name"]: RemoteEvaluator.model_validate(row) for row in rows}

evalkit/client/sessions.py

@@ -0,0 +1,17 @@
+from evalkit.client.api import ApiRequestError, AsyncApi, encode
+from evalkit.core.traces.sessions import SessionTooLarge, SessionTrajectory
+
+
+class HttpSessionReader:
+    def __init__(self, api: AsyncApi):
+        self._api = api
+
+    async def trajectory(self, agent: str, session_id: str) -> SessionTrajectory:
+        path = f"/api/agents/{encode(agent)}/sessions/{encode(session_id)}/trajectory"
+        try:
+            resp = await self._api.get(path)
+        except ApiRequestError as exc:
+            if exc.status == 413:
+                raise SessionTooLarge(exc.detail) from exc
+            raise
+        return SessionTrajectory.model_validate(resp.json())

evalkit/client/store.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import os
+import socket
+
+from evalkit.client.api import AsyncApi, encode
+from evalkit.contracts.enums import ExperimentStatus
+from evalkit.core.runner import LEASE_SECONDS, ExperimentRequest
+from evalkit.store.base import (
+    AgentRunRecord,
+    CaseRep,
+    CaseView,
+    EvaluationRow,
+    Experiment,
+    RunDetail,
+    RunForScoring,
+    RunRef,
+    Selection,
+)
+
+RENEW_TIMEOUT_SECONDS = LEASE_SECONDS / 6
+
+
+def local_holder(username: str) -> str:
+    return f"local:{username}@{socket.gethostname()}:{os.getpid()}"
+
+
+class HttpStore:
+    def __init__(self, api: AsyncApi):
+        self._api = api
+
+    async def aclose(self) -> None:
+        await self._api.aclose()
+
+    async def cases_for(
+        self, dataset: str, selection: Selection, *, agent: str | None
+    ) -> list[CaseView]:
+        rows = (
+            await self._api.post(
+                f"/api/datasets/{encode(dataset)}/cases/select",
+                json={
+                    "selection": selection.model_dump(exclude_none=True),
+                    "agent": agent,
+                },
+                retry=True,
+            )
+        ).json()
+        return [CaseView.model_validate(r) for r in rows]
+
+    async def start_experiment(self, req: ExperimentRequest) -> tuple[Experiment, list[str]]:
+        body = req.model_dump(mode="json", exclude_none=True)
+        created = (await self._api.post("/api/experiments", json=body)).json()
+        return Experiment.model_validate(created), list(created.get("dropped_cases") or [])
+
+    async def get_experiment(self, exp_id: int) -> Experiment:
+        return Experiment.model_validate((await self._api.get(f"/api/experiments/{exp_id}")).json())
+
+    async def pinned_cases(self, exp_id: int) -> list[CaseView]:
+        rows = (await self._api.get(f"/api/experiments/{exp_id}/cases")).json()
+        return [CaseView.model_validate(r) for r in rows]
+
+    async def transition(
+        self, exp_id: int, to: ExperimentStatus, holder: str | None, error: str | None = None
+    ) -> Experiment:
+        body = {"status": to.value, "holder": holder, "error": error}
+        resp = await self._api.patch(f"/api/experiments/{exp_id}", json=body, retry=True)
+        return Experiment.model_validate(resp.json())
+
+    async def renew_lease(self, exp_id: int, holder: str) -> bool:
+        resp = await self._api.post(
+            f"/api/experiments/{exp_id}/lease",
+            json={"holder": holder},
+            retry=True,
+            timeout=RENEW_TIMEOUT_SECONDS,
+        )
+        return bool(resp.json()["renewed"])
+
+    async def set_model(self, exp_id: int, model: str) -> None:
+        await self._api.patch(f"/api/experiments/{exp_id}", json={"model": model}, retry=True)
+
+    async def pending_runs(self, exp_id: int) -> list[CaseRep]:
+        rows = (await self._api.get(f"/api/experiments/{exp_id}/pending")).json()
+        return [CaseRep.model_validate(r) for r in rows]
+
+    async def harness_run_id_used(self, exp_id: int, harness_run_id: str) -> str | None:
+        resp = await self._api.get(
+            f"/api/experiments/{exp_id}/harness-run-id", params={"value": harness_run_id}
+        )
+        return resp.json()["used"]
+
+    async def record_run(
+        self,
+        exp_id: int,
+        case_id: str,
+        repetition: int,
+        run: AgentRunRecord,
+        *,
+        holder: str | None = None,
+    ) -> RunRef:
+        resp = await self._api.post(
+            f"/api/experiments/{exp_id}/runs",
+            json={
+                "case_id": case_id,
+                "repetition": repetition,
+                "run": run.model_dump(mode="json"),
+                "holder": holder,
+            },
+        )
+        return RunRef.model_validate(resp.json())
+
+    async def runs_to_score(self, exp_id: int, *, force: bool) -> list[RunForScoring]:
+        rows = (
+            await self._api.get(
+                f"/api/experiments/{exp_id}/runs-to-score", params={"force": str(force).lower()}
+            )
+        ).json()
+        return [RunForScoring.model_validate(r) for r in rows]
+
+    async def publish_evaluations(
+        self,
+        experiment_run_id: int,
+        rows: list[EvaluationRow],
+        *,
+        force: bool,
+        holder: str | None = None,
+    ) -> None:
+        await self._api.post(
+            f"/api/experiment-runs/{experiment_run_id}/evaluations",
+            json={
+                "rows": [r.model_dump(mode="json") for r in rows],
+                "force": force,
+                "holder": holder,
+            },
+            retry=True,
+        )
+
+    async def run_details(self, exp_id: int) -> list[RunDetail]:
+        rows = (await self._api.get(f"/api/experiments/{exp_id}/runs")).json()
+        return [RunDetail.model_validate(r) for r in rows]

evalkit/client/summary.py

@@ -0,0 +1,46 @@
+def _fmt(value: float | None) -> str:
+    return f"{value:.2f}" if value is not None else "  - "
+
+
+def _num(value: float | None) -> str:
+    if value is None:
+        return "-"
+    if value and abs(value) < 1:
+        return f"{value:.3g}"
+    return f"{value:.2f}".rstrip("0").rstrip(".")
+
+
+def summary_lines(report: dict) -> list[str]:
+    lines = [f"\nexperiment {report['experiment_id']}  {report['name']} of {report['dataset']}"]
+    scored = {bucket: value for bucket, value in report["buckets"].items() if value is not None}
+    if scored:
+        lines.append("buckets:")
+        lines += [f"  {bucket:<14} {_fmt(value)}" for bucket, value in scored.items()]
+    lines.append("evaluators:")
+    for stat in report["evaluators"]:
+        errored = f", {stat['errors']} errored" if stat.get("errors") else ""
+        noun = "case" if stat["cases"] == 1 else "cases"
+        counts = f"({stat['cases']} {noun}{errored})"
+        lines.append(f"  {stat['name']:<40} {_fmt(stat['score'])}  {counts}")
+    failing = [c for c in report["cases"] if c["overall"] is not None and c["overall"] < 0.8]
+    if failing:
+        lines.append("failing cases (< 0.80):")
+        for case in failing:
+            delta = f"  Δ{case['delta']:+.2f}" if case.get("delta") is not None else ""
+            lines.append(f"  {case['case_id']:<48} {_fmt(case['overall'])}{delta}")
+    errored_cases = [c for c in report["cases"] if c["errors"] or c["run_errors"]]
+    if errored_cases:
+        lines.append("errored cases:")
+        for case in errored_cases:
+            problems = [("run", error) for error in case["run_errors"]]
+            problems += sorted(case["errors"].items())
+            for name, error in problems:
+                first_line = error.splitlines()[0][:120] if error else ""
+                lines.append(f"  {case['case_id']:<48} {name}: {first_line}")
+    if report["gauges"]:
+        lines.append("gauges:")
+        lines += [
+            f"  {gauge['name']:<32} mean={_num(gauge['mean'])} median={_num(gauge['median'])}"
+            for gauge in report["gauges"]
+        ]
+    return lines