kimpton-evalrouter-sdk 0.1.0__py3-none-any.whl

kimpton_evalrouter/__init__.py

@@ -0,0 +1,92 @@
+"""Typed API client. This package is independent of the API and native runners."""
+
+from functools import cached_property as _cached_property
+
+from . import types
+from ._resources import Catalog as _Catalog
+from ._resources import Connections as _Connections
+from ._resources import Evals as _Evals
+from ._resources import Evaluations as _Evaluations
+from ._resources import Quotes as _Quotes
+from ._resources import Runs as _Runs
+from ._transport import APIError, ClientError, RequestOptions, RequestTimeout, WaitCancelled
+from ._transport import Transport as _Transport
+
+__all__ = [
+    "Client",
+    "EvalRouter",
+    "APIError",
+    "ClientError",
+    "RequestOptions",
+    "RequestTimeout",
+    "WaitCancelled",
+    "types",
+]
+__version__ = "0.1.0"
+
+
+class Client(_Transport):
+    @_cached_property
+    def catalog(self) -> _Catalog:
+        return _Catalog(self)
+
+    @_cached_property
+    def connections(self) -> _Connections:
+        return _Connections(self)
+
+    @_cached_property
+    def quotes(self) -> _Quotes:
+        return _Quotes(self)
+
+    @_cached_property
+    def runs(self) -> _Runs:
+        return _Runs(self)
+
+    @_cached_property
+    def evals(self) -> _Evals:
+        return _Evals(self)
+
+    @_cached_property
+    def evaluations(self) -> _Evaluations:
+        return _Evaluations(self)
+
+    def run(
+        self,
+        *,
+        model: str,
+        eval: str,
+        budget_usd: str | float | None = None,
+        max_charge_microusd: str | None = None,
+        provider: str | None = None,
+        coverage: dict | None = None,
+        name: str = "Evaluation",
+        metadata: dict[str, str] | None = None,
+        idempotency_key: str | None = None,
+        wait: bool = True,
+        timeout: float = 3600,
+        poll_interval: float = 2,
+    ) -> "types.RunRecord":
+        """The canonical call: route `eval` to whichever provider owns it and run it.
+
+        `model` is a managed route id or `connection:<uuid>`;
+        `eval` is `eval://provider/name/version` or `name@version`. Returns the
+        terminal run when `wait` is true, otherwise the admitted run.
+        """
+        from uuid import uuid4
+
+        body: dict = {"model": model, "eval": eval, "name": name, "metadata": metadata or {}}
+        if provider:
+            body["provider"] = provider
+        if coverage:
+            body["coverage"] = coverage
+        if max_charge_microusd is not None:
+            body["max_charge_microusd"] = max_charge_microusd
+        else:
+            body["budget_usd"] = str(budget_usd)
+        run = self.evaluations.create(body, idempotency_key=idempotency_key or f"run-{uuid4()}")
+        if not wait:
+            return run
+        return self.runs.wait(str(run["id"]), timeout=timeout, poll_interval=poll_interval)
+
+
+EvalRouter = Client

kimpton_evalrouter/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

kimpton_evalrouter/_boundary.py

@@ -0,0 +1,50 @@
+"""Validate supported evaluation inputs before issuing a request."""
+
+from ._transport import ClientError
+
+
+def validate_submission(resource, body):
+    allowed = (
+        {"selection", "model", "coverage", "max_charge_microusd"}
+        if resource == "quotes"
+        else {
+            "model",
+            "eval",
+            "budget_usd",
+            "max_charge_microusd",
+            "provider",
+            "coverage",
+            "name",
+            "metadata",
+            "split",
+        }
+    )
+    if not isinstance(body, dict) or set(body) - allowed:
+        raise ClientError(
+            "unsupported_feature", "This submission contains unsupported input fields."
+        )
+    model = body.get("model")
+    if isinstance(model, str) and resource == "evaluations":
+        valid = bool(model) and not model.startswith(("checkpoint:", "agent:", "artifact:"))
+    elif isinstance(model, dict):
+        keys = {"managed": {"kind", "route_id"}, "connection": {"kind", "connection_id"}}
+        kind = model.get("kind")
+        valid = isinstance(kind, str) and kind in keys and set(model) == keys[kind]
+    else:
+        valid = False
+    if resource == "quotes":
+        selection = body.get("selection")
+        valid = (
+            valid
+            and isinstance(selection, dict)
+            and (
+                set(selection) == {"profile_ids"}
+                or "eval" in selection
+                and not set(selection) - {"eval", "provider", "split"}
+            )
+        )
+    if not valid:
+        raise ClientError(
+            "unsupported_feature",
+            "Choose an admitted eval and a managed model or checked connection.",
+        )

kimpton_evalrouter/_helpers.py

@@ -0,0 +1,101 @@
+"""Local waiting and explicit export destinations; neither starts evaluations."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from . import types
+from ._transport import ClientError, RequestOptions, Resource, checkpoint, pause, positive
+
+TERMINAL = {"completed", "partial", "failed", "cancelled"}
+STATES = TERMINAL | {"queued", "running", "cancelling", "finalizing"}
+
+
+class RunHelpers(Resource):
+    def get(self, run_id: str, *, options: RequestOptions | None = None) -> types.RunRecord:
+        raise NotImplementedError
+
+    def export(
+        self,
+        run_id: str,
+        *,
+        params: types.RunsExportParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> bytes:
+        raise NotImplementedError
+
+    def wait(
+        self,
+        run_id: str,
+        *,
+        timeout: float = 3600,
+        poll_interval: float = 2,
+        cancel_event: threading.Event | None = None,
+        on_progress: Callable[[types.RunRecord], None] | None = None,
+    ) -> types.RunRecord:
+        deadline = time.monotonic() + positive(timeout, "Wait timeout")
+        interval = positive(poll_interval, "Poll interval")
+        while True:
+            remaining = checkpoint(deadline, cancel_event)
+            run = self.get(
+                run_id, options=RequestOptions(timeout=remaining, cancel_event=cancel_event)
+            )
+            checkpoint(deadline, cancel_event)
+            if run.get("status") not in STATES:
+                raise ClientError("invalid_response", "API returned an unknown evaluation state.")
+            if on_progress:
+                on_progress(run)
+            checkpoint(deadline, cancel_event)
+            if run["status"] in TERMINAL:
+                return run
+            pause(interval, deadline, cancel_event)
+
+    def export_to_file(
+        self,
+        run_id: str,
+        destination: str | Path,
+        *,
+        params: types.RunsExportParams | None = None,
+        options: RequestOptions | None = None,
+        overwrite: bool = False,
+    ) -> Path:
+        path = Path(destination)
+        if not overwrite and path.exists():
+            raise ClientError("destination_exists", "Export destination already exists.")
+        data = self.export(run_id, params=params, options=options)
+        return save_export(data, path, overwrite=overwrite)
+
+
+def save_export(data: bytes, destination: str | Path, *, overwrite: bool = False) -> Path:
+    path = Path(destination)
+    if not overwrite and path.exists():
+        raise ClientError("destination_exists", "Export destination already exists.")
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            dir=path.parent, prefix=".kimpton-export-", delete=False
+        ) as stream:
+            temporary = Path(stream.name)
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        if overwrite:
+            os.replace(temporary, path)
+        else:
+            os.link(temporary, path)
+            temporary.unlink()
+        return path
+    except FileExistsError:
+        raise ClientError("destination_exists", "Export destination already exists.") from None
+    except OSError:
+        raise ClientError(
+            "export_write_failed", "Could not write the export destination."
+        ) from None
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)

kimpton_evalrouter/_resources.py

@@ -0,0 +1,330 @@
+"""Generated resource methods. Edit OpenAPI and client-operations.json."""
+
+from __future__ import annotations
+
+from typing import cast
+
+from . import types
+from ._boundary import validate_submission
+from ._helpers import RunHelpers
+from ._transport import RequestOptions, Resource, segment
+
+
+class Catalog(Resource):
+    def evals(
+        self,
+        *,
+        params: types.CatalogEvalsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.Page_EvalRecord_:
+        return cast(
+            types.Page_EvalRecord_,
+            self._client.request("GET", "/evals", retry=True, options=options, query=params),
+        )
+
+    def eval(self, ref: str, *, options: RequestOptions | None = None) -> types.EvalRecord:
+        return cast(
+            types.EvalRecord,
+            self._client.request("GET", f"/evals/{segment(ref)}", retry=True, options=options),
+        )
+
+    def benchmarks(
+        self,
+        *,
+        params: types.CatalogBenchmarksParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.Page_BenchmarkRecord_:
+        return cast(
+            types.Page_BenchmarkRecord_,
+            self._client.request(
+                "GET", "/catalog/benchmarks", retry=True, options=options, query=params
+            ),
+        )
+
+    def benchmark(
+        self, slug: str, *, options: RequestOptions | None = None
+    ) -> types.BenchmarkDetail:
+        return cast(
+            types.BenchmarkDetail,
+            self._client.request(
+                "GET", f"/catalog/benchmarks/{segment(slug)}", retry=True, options=options
+            ),
+        )
+
+    def models(
+        self,
+        *,
+        params: types.CatalogModelsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.Page_ManagedRoute_:
+        return cast(
+            types.Page_ManagedRoute_,
+            self._client.request(
+                "GET", "/catalog/models", retry=True, options=options, query=params
+            ),
+        )
+
+    def execution_options(self, *, options: RequestOptions | None = None) -> types.ExecutionOptions:
+        return cast(
+            types.ExecutionOptions,
+            self._client.request("GET", "/catalog/execution-options", retry=True, options=options),
+        )
+
+    def environments(
+        self,
+        *,
+        params: types.CatalogEnvironmentsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.Page_EnvironmentRecord_:
+        return cast(
+            types.Page_EnvironmentRecord_,
+            self._client.request(
+                "GET", "/catalog/environments", retry=True, options=options, query=params
+            ),
+        )
+
+    def environment(
+        self, family: str, *, options: RequestOptions | None = None
+    ) -> types.EnvironmentFamily:
+        return cast(
+            types.EnvironmentFamily,
+            self._client.request(
+                "GET", f"/catalog/environments/{segment(family)}", retry=True, options=options
+            ),
+        )
+
+
+class Connections(Resource):
+    def list(
+        self,
+        *,
+        params: types.ConnectionsListParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.Page_ConnectionRecord_:
+        return cast(
+            types.Page_ConnectionRecord_,
+            self._client.request("GET", "/connections", retry=True, options=options, query=params),
+        )
+
+    def create(
+        self, body: types.NewConnection, *, options: RequestOptions | None = None
+    ) -> types.ConnectionRecord:
+        return cast(
+            types.ConnectionRecord,
+            self._client.request("POST", "/connections", retry=False, options=options, body=body),
+        )
+
+    def check(
+        self, connection_id: str, *, options: RequestOptions | None = None
+    ) -> types.ConnectionRecord:
+        return cast(
+            types.ConnectionRecord,
+            self._client.request(
+                "POST", f"/connections/{segment(connection_id)}/check", retry=False, options=options
+            ),
+        )
+
+    def update(
+        self,
+        connection_id: str,
+        body: types.UpdateConnection,
+        *,
+        options: RequestOptions | None = None,
+    ) -> types.ConnectionRecord:
+        return cast(
+            types.ConnectionRecord,
+            self._client.request(
+                "PATCH",
+                f"/connections/{segment(connection_id)}",
+                retry=False,
+                options=options,
+                body=body,
+            ),
+        )
+
+    def disable(
+        self, connection_id: str, *, options: RequestOptions | None = None
+    ) -> types.OKResponse:
+        return cast(
+            types.OKResponse,
+            self._client.request(
+                "DELETE", f"/connections/{segment(connection_id)}", retry=False, options=options
+            ),
+        )
+
+
+class Quotes(Resource):
+    def create(
+        self, body: types.NewQuote, *, options: RequestOptions | None = None
+    ) -> types.QuoteRecord:
+        validate_submission("quotes", body)
+        return cast(
+            types.QuoteRecord,
+            self._client.request("POST", "/quotes", retry=False, options=options, body=body),
+        )
+
+    def get(self, quote_id: str, *, options: RequestOptions | None = None) -> types.QuoteRecord:
+        return cast(
+            types.QuoteRecord,
+            self._client.request(
+                "GET", f"/quotes/{segment(quote_id)}", retry=True, options=options
+            ),
+        )
+
+
+class Runs(RunHelpers):
+    def create(
+        self, body: types.NewRun, *, idempotency_key: str, options: RequestOptions | None = None
+    ) -> types.RunRecord:
+        return cast(
+            types.RunRecord,
+            self._client.request(
+                "POST",
+                "/runs",
+                retry=True,
+                options=options,
+                body=body,
+                idempotency_key=idempotency_key,
+            ),
+        )
+
+    def list(
+        self, *, params: types.RunsListParams | None = None, options: RequestOptions | None = None
+    ) -> types.Page_RunRecord_:
+        return cast(
+            types.Page_RunRecord_,
+            self._client.request("GET", "/runs", retry=True, options=options, query=params),
+        )
+
+    def get(self, run_id: str, *, options: RequestOptions | None = None) -> types.RunRecord:
+        return cast(
+            types.RunRecord,
+            self._client.request("GET", f"/runs/{segment(run_id)}", retry=True, options=options),
+        )
+
+    def receipt(self, run_id: str, *, options: RequestOptions | None = None) -> types.RunReceipt:
+        return cast(
+            types.RunReceipt,
+            self._client.request(
+                "GET", f"/runs/{segment(run_id)}/receipt", retry=True, options=options
+            ),
+        )
+
+    def executions(
+        self,
+        run_id: str,
+        *,
+        params: types.RunsExecutionsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.SandboxExecutionPage:
+        return cast(
+            types.SandboxExecutionPage,
+            self._client.request(
+                "GET",
+                f"/runs/{segment(run_id)}/executions",
+                retry=True,
+                options=options,
+                query=params,
+            ),
+        )
+
+    def events(
+        self,
+        run_id: str,
+        *,
+        params: types.RunsEventsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.EventPage:
+        return cast(
+            types.EventPage,
+            self._client.request(
+                "GET", f"/runs/{segment(run_id)}/events", retry=True, options=options, query=params
+            ),
+        )
+
+    def cancel(self, run_id: str, *, options: RequestOptions | None = None) -> types.RunRecord:
+        return cast(
+            types.RunRecord,
+            self._client.request(
+                "POST", f"/runs/{segment(run_id)}/cancel", retry=True, options=options
+            ),
+        )
+
+    def results(
+        self,
+        run_id: str,
+        *,
+        params: types.RunsResultsParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> types.ResultRecord:
+        return cast(
+            types.ResultRecord,
+            self._client.request(
+                "GET", f"/runs/{segment(run_id)}/results", retry=True, options=options, query=params
+            ),
+        )
+
+    def export(
+        self,
+        run_id: str,
+        *,
+        params: types.RunsExportParams | None = None,
+        options: RequestOptions | None = None,
+    ) -> bytes:
+        return cast(
+            bytes,
+            self._client.request(
+                "GET",
+                f"/runs/{segment(run_id)}/export",
+                retry=True,
+                options=options,
+                query=params,
+                binary=True,
+            ),
+        )
+
+
+class Evals(Resource):
+    def list(
+        self, *, params: types.EvalsListParams | None = None, options: RequestOptions | None = None
+    ) -> types.Page_EvalRecord_:
+        return cast(
+            types.Page_EvalRecord_,
+            self._client.request("GET", "/evals", retry=True, options=options, query=params),
+        )
+
+    def get(self, ref: str, *, options: RequestOptions | None = None) -> types.EvalRecord:
+        return cast(
+            types.EvalRecord,
+            self._client.request("GET", f"/evals/{segment(ref)}", retry=True, options=options),
+        )
+
+
+class Evaluations(Resource):
+    def create(
+        self,
+        body: types.NewEvaluation,
+        *,
+        idempotency_key: str,
+        options: RequestOptions | None = None,
+    ) -> types.RunRecord:
+        validate_submission("evaluations", body)
+        return cast(
+            types.RunRecord,
+            self._client.request(
+                "POST",
+                "/evaluations",
+                retry=True,
+                options=options,
+                body=body,
+                idempotency_key=idempotency_key,
+            ),
+        )
+
+    def get(self, evaluation_id: str, *, options: RequestOptions | None = None) -> types.RunRecord:
+        return cast(
+            types.RunRecord,
+            self._client.request(
+                "GET", f"/evaluations/{segment(evaluation_id)}", retry=True, options=options
+            ),
+        )