manifold-cli 0.0.39__py3-none-any.whl

manifold_cli/clients/manifold.py

@@ -0,0 +1,1288 @@
+import asyncio
+import json
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from datetime import datetime
+from http import HTTPStatus
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+from uuid import UUID
+
+import httpx
+from pydantic import TypeAdapter
+
+from manifold_cli.models.auth import CliConfig, User
+from manifold_cli.models.benchmark import (
+    Benchmark,
+    BenchmarkNotFoundError,
+    BenchmarkVersion,
+    RubricContent,
+)
+from manifold_cli.models.catalog import Adapter, CatalogRegisterError, Visibility
+from manifold_cli.models.errors import (
+    ManifoldApiError,
+    error_detail,
+    raw_error_detail,
+)
+from manifold_cli.models.log import (
+    LogBatch,
+    LogLine,
+    LogMatch,
+    LogProducer,
+    LogSource,
+    RunLogStream,
+)
+from manifold_cli.models.observability import (
+    ReportBenchmarkMetricsRequest,
+    ReportPolicyMetricsRequest,
+    ReportResourceUsageRequest,
+)
+from manifold_cli.models.organization import Organization, OrganizationNotFoundError
+from manifold_cli.models.pagination import Page, SortOrder
+from manifold_cli.models.policy import Policy, PolicyNotFoundError, PolicyVersion
+from manifold_cli.models.run import (
+    Assignment,
+    AssignmentDispatch,
+    AssignmentNotAssignedToRunnerError,
+    Episode,
+    EpisodeReport,
+    EpisodesRejectedError,
+    PublishedLiveView,
+    ReplayRejectedError,
+    ReplayUploadSlot,
+    Run,
+    RunBridge,
+    RunCancelRejectedError,
+    RunDeleteRejectedError,
+    RunNotFoundError,
+    RunSideRefusedError,
+    RunStatus,
+    RunSubmitError,
+    SchedulingType,
+    SideHeartbeat,
+    SideOutcome,
+)
+from manifold_cli.models.runner import (
+    GpuModel,
+    InstanceOverrideType,
+    ListedRunner,
+    MintedRunner,
+    Runner,
+    RunnerGpuDevice,
+    RunnerNameTakenError,
+    RunnerNotFoundError,
+    RunnerProvider,
+    RunnerSource,
+)
+from manifold_cli.models.task_queue import (
+    TaskClaim,
+    TaskQueueLeaseLostError,
+    TaskResult,
+)
+from manifold_cli.models.token import (
+    AccessToken,
+    AccessTokenNotFoundError,
+    CreatedAccessToken,
+    Scope,
+)
+
+logger = logging.getLogger(__name__)
+
+# The recording's own media type, signed into the slot, so the PUT has to send it.
+RRD_MIME = "application/vnd.rerun.rrd"
+
+# A recording runs to megabytes and the upload is a single request, so it gets
+# its own generous timeout rather than the client's default.
+REPLAY_PUT_TIMEOUT = 120.0
+HEARTBEAT_TIMEOUT_SECONDS = 5.0
+# An episode report is a write the server keeps whether or not the reply is
+# read, so it waits well past httpx's five-second default before giving up.
+EPISODE_REPORT_TIMEOUT_SECONDS = 30.0
+
+_ACCESS_TOKENS_ADAPTER = TypeAdapter(list[AccessToken])
+
+
+class AssignmentHeartbeatClient:
+    def __init__(self, base_url: str, client: httpx.AsyncClient):
+        self._base_url = base_url
+        self._client = client
+
+    async def send(self, assignment: Assignment, *, serving: bool) -> SideHeartbeat:
+        """Report this assignment's side alive, and serving when `serving` is set.
+
+        `assignment` is the attempt returned by the dispatch. Including it in
+        the request prevents the server from updating a later attempt if the
+        request completes after the first attempt ends.
+
+        A heartbeat that is not serving is the daemon's liveness until the
+        policy has accepted the benchmark's signature: the server marks the
+        runner seen without changing its preparation or startup phase.
+        The first serving heartbeat sets the side to `running`.
+
+        The answer carries how the peer finished, which is the other instruction
+        a side gets: a policy container serves until it is stopped, so its own
+        worker ending does not mean the run is over.
+
+        Raises `AssignmentNotAssignedToRunnerError` when the attempt is no longer this
+        runner's. That refusal is the instruction to tear the worker down.
+        """
+        async with asyncio.timeout(HEARTBEAT_TIMEOUT_SECONDS):
+            response = await self._client.post(
+                f"{self._base_url}/v0/runs/{assignment.run_id}"
+                f"/attempts/{assignment.id}/heartbeat",
+                params={"serving": serving},
+            )
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise _heartbeat_refusal(response, assignment.id)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return SideHeartbeat.model_validate(response.json())
+
+
+class ManifoldApiClient:
+    def __init__(self, base_url: str, auth: httpx.Auth | None = None):
+        self._base_url = base_url.rstrip("/")
+        self._auth = auth
+        self._client = httpx.Client(auth=auth)
+
+    @property
+    def base_url(self) -> str:
+        """The deployment this client sends its requests to.
+
+        The policy side reads it to work out which of its own addresses a peer
+        dials: the interface that routes to this URL is the one a peer reaches.
+        """
+        return self._base_url
+
+    def close(self) -> None:
+        self._client.close()
+
+    def get_cli_config(self) -> CliConfig:
+        # Unauthenticated bootstrap: this endpoint needs no token, and it is
+        # also what the auth flow fetches to refresh one — routing it through
+        # auth would recurse. auth=None opts this request out of the client's
+        # auth flow entirely.
+        response = self._client.get(f"{self._base_url}/auth/cli/config", auth=None)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return CliConfig.model_validate(response.json())
+
+    def get_me(self) -> User:
+        response = self._client.get(f"{self._base_url}/auth/me")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return User.model_validate(response.json())
+
+    def list_organizations(self) -> list[Organization]:
+        response = self._client.get(f"{self._base_url}/organization")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [Organization.model_validate(item) for item in response.json()]
+
+    def list_policies(
+        self, *, limit: int | None = None, cursor: str | None = None
+    ) -> Page[Policy]:
+        response = self._client.get(
+            f"{self._base_url}/v0/policies", params=_page_params(limit, cursor)
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Page[Policy].model_validate(response.json())
+
+    def list_benchmarks(
+        self, *, limit: int | None = None, cursor: str | None = None
+    ) -> Page[Benchmark]:
+        response = self._client.get(
+            f"{self._base_url}/v0/benchmarks", params=_page_params(limit, cursor)
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Page[Benchmark].model_validate(response.json())
+
+    def list_policy_versions(self, slug: str, organization: str) -> list[PolicyVersion]:
+        response = self._client.get(
+            f"{self._base_url}/v0/policies/{slug}/versions",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise PolicyNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [PolicyVersion.model_validate(item) for item in response.json()]
+
+    def list_benchmark_versions(
+        self, slug: str, organization: str
+    ) -> list[BenchmarkVersion]:
+        response = self._client.get(
+            f"{self._base_url}/v0/benchmarks/{slug}/versions",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise BenchmarkNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [BenchmarkVersion.model_validate(item) for item in response.json()]
+
+    def upsert_policy(
+        self,
+        slug: str,
+        organization: str,
+        *,
+        display_name: str,
+        visibility: Visibility,
+    ) -> None:
+        response = self._client.put(
+            f"{self._base_url}/v0/policies/{slug}",
+            json={
+                "organization": organization,
+                "display_name": display_name,
+                "visibility": visibility.value,
+            },
+        )
+        _raise_for_register_rejection(response, slug)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def register_policy_version(
+        self,
+        slug: str,
+        version: str,
+        organization: str,
+        *,
+        adapter: Adapter,
+        config: dict[str, Any],
+        minimum_gpu_memory_gb: int,
+        minimum_driver_major: int | None,
+        maximum_driver_major: int | None,
+        minimum_gpu_count: int | None = None,
+    ) -> PolicyVersion:
+        body = {
+            "organization": organization,
+            "adapter": adapter.value,
+            "config": config,
+            "minimum_gpu_memory_gb": minimum_gpu_memory_gb,
+        }
+        if minimum_driver_major is not None:
+            body["minimum_driver_major"] = minimum_driver_major
+        if maximum_driver_major is not None:
+            body["maximum_driver_major"] = maximum_driver_major
+        if minimum_gpu_count is not None:
+            body["minimum_gpu_count"] = minimum_gpu_count
+        response = self._client.put(
+            f"{self._base_url}/v0/policies/{slug}/versions/{version}",
+            json=body,
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise PolicyNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        _raise_for_register_rejection(response, slug)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return PolicyVersion.model_validate(response.json())
+
+    def delete_policy(self, slug: str, organization: str) -> None:
+        response = self._client.delete(
+            f"{self._base_url}/v0/policies/{quote(slug, safe='')}",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise PolicyNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def delete_policy_version(self, slug: str, version: str, organization: str) -> None:
+        response = self._client.delete(
+            f"{self._base_url}/v0/policies/{quote(slug, safe='')}"
+            f"/versions/{quote(version, safe='')}",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise PolicyNotFoundError(
+                error_detail(response.text) or f"'{slug}:{version}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def upsert_benchmark(
+        self,
+        slug: str,
+        organization: str,
+        *,
+        display_name: str,
+        visibility: Visibility,
+    ) -> None:
+        response = self._client.put(
+            f"{self._base_url}/v0/benchmarks/{slug}",
+            json={
+                "organization": organization,
+                "display_name": display_name,
+                "visibility": visibility.value,
+            },
+        )
+        _raise_for_register_rejection(response, slug)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def register_benchmark_version(
+        self,
+        slug: str,
+        version: str,
+        organization: str,
+        *,
+        adapter: Adapter,
+        config: dict[str, Any],
+        episodes: int,
+        shards: int,
+        shards_per_gpu: int | None,
+        minimum_gpu_memory_gb: int,
+        minimum_driver_major: int | None,
+        maximum_driver_major: int | None,
+        minimum_gpu_count: int | None = None,
+        scheduling_type: SchedulingType,
+        allow_preemptible: bool,
+    ) -> BenchmarkVersion:
+        body = {
+            "organization": organization,
+            "adapter": adapter.value,
+            "config": config,
+            "episodes": episodes,
+            "shards": shards,
+            "minimum_gpu_memory_gb": minimum_gpu_memory_gb,
+            "scheduling_type": scheduling_type.value,
+            "allow_preemptible": allow_preemptible,
+            "shards_per_gpu": shards_per_gpu,
+        }
+        if minimum_driver_major is not None:
+            body["minimum_driver_major"] = minimum_driver_major
+        if maximum_driver_major is not None:
+            body["maximum_driver_major"] = maximum_driver_major
+        if minimum_gpu_count is not None:
+            body["minimum_gpu_count"] = minimum_gpu_count
+        response = self._client.put(
+            f"{self._base_url}/v0/benchmarks/{slug}/versions/{version}",
+            json=body,
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise BenchmarkNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        _raise_for_register_rejection(response, slug)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return BenchmarkVersion.model_validate(response.json())
+
+    def publish_benchmark_rubric(
+        self,
+        slug: str,
+        version: str,
+        organization: str,
+        *,
+        rubric: RubricContent,
+    ) -> None:
+        """Publish `rubric` as the benchmark version's rubric.
+
+        The client sends the groups and tasks exactly as it read them from the
+        image. The server replaces any stored rubric and checks each group and
+        task.
+        """
+        response = self._client.put(
+            f"{self._base_url}/v0/benchmarks/{slug}/versions/{version}/rubric",
+            json={
+                "organization": organization,
+                "groups": rubric.groups,
+                "tasks": rubric.tasks,
+                "grading": rubric.grading,
+            },
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise BenchmarkNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        _raise_for_register_rejection(response, slug)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def delete_benchmark(self, slug: str, organization: str) -> None:
+        response = self._client.delete(
+            f"{self._base_url}/v0/benchmarks/{quote(slug, safe='')}",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise BenchmarkNotFoundError(
+                error_detail(response.text) or f"'{slug}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def delete_benchmark_version(
+        self, slug: str, version: str, organization: str
+    ) -> None:
+        response = self._client.delete(
+            f"{self._base_url}/v0/benchmarks/{quote(slug, safe='')}"
+            f"/versions/{quote(version, safe='')}",
+            params={"organization": organization},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise BenchmarkNotFoundError(
+                error_detail(response.text) or f"'{slug}:{version}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def submit_run(
+        self,
+        organization: str,
+        policy: str,
+        benchmark: str,
+        *,
+        name: str,
+        shards: int | None,
+        shards_per_gpu: int | None,
+        policy_version: str | None = None,
+        benchmark_version: str | None = None,
+    ) -> Run:
+        body: dict[str, Any] = {
+            "organization": organization,
+            "policy": policy,
+            "benchmark": benchmark,
+            "name": name,
+            "shards_per_gpu": shards_per_gpu,
+        }
+        # Left out rather than sent as null: an absent key leaves the default
+        # to the server, which takes the newest version and the benchmark
+        # version's own shard count.
+        if policy_version is not None:
+            body["policy_version"] = policy_version
+        if benchmark_version is not None:
+            body["benchmark_version"] = benchmark_version
+        if shards is not None:
+            body["shards"] = shards
+        response = self._client.post(f"{self._base_url}/v0/runs", json=body)
+        # A 403 here is the server refusing the pairing itself: a private policy
+        # or benchmark owned by an organization other than the run's. A 409
+        # means that the submitter does not serve the self-hosted version. Both
+        # are the caller's to fix, not an API fault.
+        if response.status_code in (
+            HTTPStatus.NOT_FOUND,
+            HTTPStatus.FORBIDDEN,
+            HTTPStatus.CONFLICT,
+        ):
+            raise RunSubmitError(
+                error_detail(response.text) or "The run could not be submitted."
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Run.model_validate(response.json())
+
+    def list_runs(
+        self, organization: str, *, limit: int | None = None, cursor: str | None = None
+    ) -> Page[Run]:
+        response = self._client.get(
+            f"{self._base_url}/v0/runs",
+            params={"organization": organization, **_page_params(limit, cursor)},
+        )
+        # NOTE: the runs themselves are filtered, not looked up, so the only
+        # thing this route can fail to find is the organization.
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            # The fallback message deliberately does not distinguish a slug
+            # owned by someone else from one that does not exist.
+            raise OrganizationNotFoundError(
+                error_detail(response.text) or f"'{organization}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Page[Run].model_validate(response.json())
+
+    def get_run(self, run_id: UUID) -> Run:
+        response = self._client.get(f"{self._base_url}/v0/runs/{run_id}")
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Run.model_validate(response.json())
+
+    def cancel_run(self, run_id: UUID) -> Run:
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/status",
+            json={"status": "canceled"},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        # A 409 means the run is already in a terminal state, possibly since
+        # the caller last read it.
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise RunCancelRejectedError(
+                error_detail(response.text) or "the run cannot be canceled"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Run.model_validate(response.json())
+
+    def delete_run(self, run_id: UUID) -> None:
+        response = self._client.delete(f"{self._base_url}/v0/runs/{run_id}")
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise RunDeleteRejectedError(
+                error_detail(response.text) or "the run cannot be deleted"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def poll_for_assignment(self, *, timeout_s: float) -> AssignmentDispatch | None:
+        """Wait for the assignment naming this runner, and collect it.
+
+        The runner does not choose: a placement pass writes the row before this
+        is called, and the poll hands over one side or nothing. None means the
+        request was held open and nothing arrived, which is the ordinary answer
+        for an idle runner -- ask again.
+
+        `timeout_s` has to exceed the server's own poll window, or every poll
+        looks like a transport failure from here.
+        """
+        response = self._client.get(
+            f"{self._base_url}/v0/runs/assignment",
+            timeout=timeout_s,
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        body = response.json()
+        return AssignmentDispatch.model_validate(body) if body is not None else None
+
+    def claim_task(
+        self, run_id: UUID, assignment_id: UUID, claim_request_id: UUID
+    ) -> TaskClaim:
+        response = self._task_queue_request(
+            run_id,
+            "claim",
+            {
+                "assignment_id": str(assignment_id),
+                "claim_request_id": str(claim_request_id),
+            },
+        )
+        return TaskClaim.model_validate(response.json())
+
+    def mint_task_upload(
+        self,
+        run_id: UUID,
+        assignment_id: UUID,
+        attempt_id: UUID,
+        episode_idx: int,
+        size_bytes: int,
+    ) -> ReplayUploadSlot:
+        response = self._task_queue_request(
+            run_id,
+            "mint-upload",
+            {
+                "assignment_id": str(assignment_id),
+                "attempt_id": str(attempt_id),
+                "episode_idx": episode_idx,
+                "size_bytes": size_bytes,
+            },
+        )
+        return ReplayUploadSlot.model_validate(response.json())
+
+    def complete_task(
+        self,
+        run_id: UUID,
+        assignment_id: UUID,
+        attempt_id: UUID,
+        results: list[TaskResult],
+    ) -> None:
+        self._task_queue_request(
+            run_id,
+            "complete",
+            {
+                "assignment_id": str(assignment_id),
+                "attempt_id": str(attempt_id),
+                "results": [result.model_dump(mode="json") for result in results],
+            },
+        )
+
+    def _task_queue_request(
+        self, run_id: UUID, operation: str, body: dict[str, Any]
+    ) -> httpx.Response:
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/task-queue/{operation}",
+            json=body,
+            timeout=EPISODE_REPORT_TIMEOUT_SECONDS,
+        )
+        if (
+            response.status_code == HTTPStatus.CONFLICT
+            and _refusal_code(response.text) == "lease_lost"
+        ):
+            raise TaskQueueLeaseLostError(response.status_code, response.text)
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return response
+
+    @asynccontextmanager
+    async def assignment_heartbeats(
+        self,
+    ) -> AsyncIterator[AssignmentHeartbeatClient]:
+        async with httpx.AsyncClient(auth=self._auth, timeout=None) as client:
+            yield AssignmentHeartbeatClient(self._base_url, client)
+
+    def begin_assignment_preparation(self) -> Assignment:
+        """Move this runner's collected side into preparation."""
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/assignment/preparation/start"
+        )
+        return Assignment.model_validate(_assignment_body(response))
+
+    def complete_assignment_preparation(self) -> Assignment:
+        """Move this runner's prepared side into worker startup."""
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/assignment/preparation/complete"
+        )
+        return Assignment.model_validate(_assignment_body(response))
+
+    def publish_run_bridge(
+        self, run_id: UUID, address: str, *, can_send_step_times: bool
+    ) -> RunBridge:
+        """Record where the benchmark side dials to reach the policy side, and
+        whether the policy side's bridge can send step times."""
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/bridge",
+            json={"address": address, "can_send_step_times": can_send_step_times},
+        )
+        return RunBridge.model_validate(_assigned_side_body(response, run_id))
+
+    def get_run_bridge(self, run_id: UUID) -> RunBridge:
+        """Read the policy side's address. None while it is still loading."""
+        response = self._client.get(f"{self._base_url}/v0/runs/{run_id}/bridge")
+        return RunBridge.model_validate(_assigned_side_body(response, run_id))
+
+    def report_run_side(
+        self,
+        run_id: UUID,
+        assignment_id: UUID,
+        outcome: SideOutcome,
+        *,
+        detail: str,
+        failed_attempt_id: UUID | None,
+    ) -> Run:
+        """Report how this runner's attempt at a side of the run finished.
+
+        `failed_attempt_id` is the task-queue attempt whose worker failed.
+        """
+        body: dict[str, Any] = {"outcome": outcome.value, "detail": detail}
+        if failed_attempt_id is not None:
+            body["failed_attempt_id"] = str(failed_attempt_id)
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/attempts/{assignment_id}/report",
+            json=body,
+        )
+        return Run.model_validate(_assigned_side_body(response, run_id))
+
+    def list_run_episodes(self, run_id: UUID) -> list[Episode]:
+        response = self._client.get(f"{self._base_url}/v0/runs/{run_id}/episodes")
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [Episode.model_validate(item) for item in response.json()]
+
+    def list_run_log_streams(self, run_id: UUID) -> list[RunLogStream]:
+        response = self._client.get(f"{self._base_url}/v0/runs/{run_id}/logs/streams")
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [RunLogStream.model_validate(item) for item in response.json()]
+
+    def search_run_log(
+        self,
+        run_id: UUID,
+        *,
+        pattern: str,
+        assignment_id: UUID | None,
+        context_lines: int | None,
+        since: datetime | None,
+        until: datetime | None,
+        order: SortOrder,
+        cursor: str | None,
+    ) -> Page[LogMatch]:
+        params: dict[str, str | int] = {"pattern": pattern, "order": order.value}
+        if assignment_id is not None:
+            params["assignment_id"] = str(assignment_id)
+        if context_lines is not None:
+            params["context_lines"] = context_lines
+        if since is not None:
+            params["since"] = since.isoformat()
+        if until is not None:
+            params["until"] = until.isoformat()
+        if cursor is not None:
+            params["cursor"] = cursor
+        response = self._client.get(
+            f"{self._base_url}/v0/runs/{run_id}/logs/search", params=params
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Page[LogMatch].model_validate(response.json())
+
+    def read_run_log(
+        self,
+        run_id: UUID,
+        *,
+        assignment_id: UUID,
+        producer: LogProducer,
+        worker_index: int,
+        source: LogSource,
+        around_seq: int | None,
+        limit: int | None,
+    ) -> list[LogLine]:
+        params: dict[str, int] = {}
+        if around_seq is not None:
+            params["around_seq"] = around_seq
+        if limit is not None:
+            params["limit"] = limit
+        response = self._client.get(
+            f"{self._base_url}/v0/runs/{run_id}/logs/streams/{assignment_id}"
+            f"/{producer.value}/{worker_index}/{source.value}",
+            params=params,
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [LogLine.model_validate(item) for item in response.json()]
+
+    def record_run_episodes(self, run_id: UUID, episodes: list[EpisodeReport]) -> Run:
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/episodes",
+            json={"episodes": [report.model_dump(mode="json") for report in episodes]},
+            timeout=EPISODE_REPORT_TIMEOUT_SECONDS,
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise EpisodesRejectedError(
+                error_detail(response.text) or "the run cannot record these episodes"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Run.model_validate(response.json())
+
+    def publish_live_view(
+        self,
+        run_id: UUID,
+        worker_index: int,
+        *,
+        episode_idx: int,
+        step: int,
+        task: str,
+        captured_at: datetime,
+        jpeg: bytes | None,
+        task_id: str | None = None,
+    ) -> PublishedLiveView:
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/live-views/{worker_index}",
+            params={
+                "episode_idx": episode_idx,
+                "step": step,
+                "task": task,
+                "captured_at": captured_at.isoformat(),
+                **({} if task_id is None else {"task_id": task_id}),
+            },
+            content=jpeg or b"",
+            headers={"content-type": "image/jpeg"} if jpeg is not None else {},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return PublishedLiveView.model_validate(response.json())
+
+    def delete_live_view(
+        self, run_id: UUID, worker_index: int, *, assignment_id: UUID
+    ) -> None:
+        """Delete the benchmark worker's live view for `assignment_id`.
+
+        The server accepts the request only while `assignment_id` is assigned
+        to this runner and no later benchmark attempt has replaced it.
+        """
+        response = self._client.delete(
+            f"{self._base_url}/v0/runs/{run_id}/live-views/{worker_index}",
+            params={"assignment_id": str(assignment_id)},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(
+                error_detail(response.text)
+                or "latest benchmark assignment does not match the request"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def post_captured_output(self, run_id: UUID, batch: LogBatch) -> None:
+        """Send one batch of the output this runner's side has captured.
+
+        Nothing comes back. The server resolves which side's output this is from
+        the credential, so a runner with one side of a run assigned to it cannot post
+        for the other.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/logs",
+            json=batch.model_dump(mode="json"),
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def mint_replay_upload(
+        self, run_id: UUID, episode_idx: int, size_bytes: int
+    ) -> ReplayUploadSlot:
+        """Ask where one episode's replay should be PUT, declaring its size.
+
+        The size is signed into the URL, so the PUT has to send exactly that many
+        bytes: the recording is written before the slot is asked for.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/replays/{episode_idx}/slot",
+            json={"size_bytes": size_bytes},
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise ReplayRejectedError(
+                error_detail(response.text) or "the run cannot take this replay"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return ReplayUploadSlot.model_validate(response.json())
+
+    def put_replay(self, slot: ReplayUploadSlot, path: Path) -> int:
+        """PUT the recording to the slot's signed URL, returning the bytes sent.
+
+        Straight to object storage rather than through the API, which is what the
+        signed URL is for. A fresh client: the signature carries its own
+        authorization, and the session's bearer token has no business leaving
+        Manifold's own host.
+        """
+        size = path.stat().st_size
+        with (
+            path.open("rb") as body,
+            httpx.Client(timeout=REPLAY_PUT_TIMEOUT) as client,
+        ):
+            response = client.put(
+                slot.url, content=body, headers={"Content-Type": RRD_MIME}
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return size
+
+    def confirm_replay_upload(self, run_id: UUID, episode_idx: int) -> None:
+        """Tell the server the PUT is done, which is what records the row.
+
+        Nothing to report: the server heads the object and reads the size off it,
+        so a confirm carries no claim the bucket cannot settle.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/runs/{run_id}/replays/{episode_idx}"
+        )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunNotFoundError(error_detail(response.text) or "run not found")
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise ReplayRejectedError(
+                error_detail(response.text) or "the run cannot take this replay"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def mint_runner(
+        self,
+        name: str,
+        *,
+        ram_gb: int | None,
+        gpu_memory_gb: int,
+        gpu_devices: list[RunnerGpuDevice],
+        gpu_model: GpuModel,
+        provider: RunnerProvider | None,
+        is_preemptible: bool,
+        organization: str | None,
+    ) -> MintedRunner:
+        # Without an organization, the server mints the runner in the operator
+        # organization.
+        body: dict[str, object] = {
+            "name": name,
+            "ram_gb": ram_gb,
+            "gpu_memory_gb": gpu_memory_gb,
+            "gpu_devices": [device.model_dump() for device in gpu_devices],
+            "gpu_model": gpu_model,
+            "is_preemptible": is_preemptible,
+        }
+        if provider is not None:
+            body["provider"] = provider
+        if organization is not None:
+            body["organization"] = organization
+        response = self._client.post(f"{self._base_url}/v0/runners", json=body)
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise RunnerNameTakenError(
+                error_detail(response.text) or f"a runner named '{name}' already exists"
+            )
+        if organization is not None and response.status_code == HTTPStatus.NOT_FOUND:
+            raise OrganizationNotFoundError(
+                error_detail(response.text) or f"'{organization}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return MintedRunner.model_validate(response.json())
+
+    def register_serving_runner(
+        self,
+        name: str,
+        organization: str,
+        *,
+        policy: str,
+        version: str,
+        gpu_devices: list[RunnerGpuDevice],
+        source: RunnerSource,
+        signature: dict[str, Any] | None,
+    ) -> MintedRunner:
+        """Register this machine as a runner for one policy version."""
+        response = self._client.post(
+            f"{self._base_url}/v0/runners/serving",
+            json={
+                "organization": organization,
+                "name": name,
+                "policy": policy,
+                "version": version,
+                "gpu_devices": [device.model_dump() for device in gpu_devices],
+                "source": source.model_dump(),
+                "signature": signature,
+            },
+        )
+        if response.status_code == HTTPStatus.CONFLICT:
+            raise RunnerNameTakenError(
+                error_detail(response.text) or f"a runner named '{name}' already exists"
+            )
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise PolicyNotFoundError(
+                error_detail(response.text) or f"policy version '{version}' not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return MintedRunner.model_validate(response.json())
+
+    def release_runner(self) -> None:
+        """Revoke this runner's own credential.
+
+        The server identifies the runner from the token.
+        """
+        response = self._client.post(f"{self._base_url}/v0/runners/release")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def start_runner(self) -> None:
+        """Mark this runner started, so that placement may assign a side to it.
+
+        The server identifies the runner from the token.
+        """
+        response = self._client.post(f"{self._base_url}/v0/runners/start")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def stop_runner(self) -> None:
+        """Mark this runner stopping, so that placement skips it.
+
+        The server identifies the runner from the token.
+        """
+        response = self._client.post(f"{self._base_url}/v0/runners/stop")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def report_driver(self, driver_version: str) -> None:
+        """Record the driver this box reports.
+
+        The server looks the runner up from the API credential, which is
+        minted per runner.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/runners/driver",
+            json={"driver_version": driver_version},
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def report_resource_usage(self, report: ReportResourceUsageRequest) -> None:
+        """Send samples of this box's resource usage for export to Honeycomb.
+
+        The server looks the runner up from the API credential.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/observability/resource-usage",
+            json=report.model_dump(mode="json"),
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def report_policy_metrics(self, report: ReportPolicyMetricsRequest) -> None:
+        """Send the policy steps that this box's bridge timed for export to
+        Honeycomb.
+
+        The server looks the runner up from the API credential.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/observability/policy-metrics",
+            json=report.model_dump(mode="json"),
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def report_benchmark_metrics(self, report: ReportBenchmarkMetricsRequest) -> None:
+        """Send the steps that this box's benchmark bridge timed for export to
+        Honeycomb.
+
+        The server looks the runner up from the API credential.
+        """
+        response = self._client.post(
+            f"{self._base_url}/v0/observability/benchmark-metrics",
+            json=report.model_dump(mode="json"),
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+
+    def list_runners(self, *, organization: str | None) -> list[ListedRunner]:
+        response = self._client.get(
+            f"{self._base_url}/v0/runners", params=_organization_params(organization)
+        )
+        if organization is not None and response.status_code == HTTPStatus.NOT_FOUND:
+            raise OrganizationNotFoundError(
+                error_detail(response.text) or f"'{organization}' was not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return [ListedRunner.model_validate(item) for item in response.json()]
+
+    def revoke_runner(self, runner_id: UUID, *, organization: str | None) -> Runner:
+        response = self._client.post(
+            f"{self._base_url}/v0/runners/{runner_id}/revoke",
+            params=_organization_params(organization),
+        )
+        # NOTE: the server also returns 404 for an unknown organization, and
+        # its detail says whether the organization or the runner was not found.
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise RunnerNotFoundError(error_detail(response.text) or "runner not found")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return Runner.model_validate(response.json())
+
+    def set_instance_override(
+        self,
+        runner_id: UUID,
+        override_type: InstanceOverrideType,
+        *,
+        until: datetime | None,
+        organization: str | None,
+    ) -> Runner:
+        response = self._client.put(
+            f"{self._base_url}/v0/runners/{runner_id}/instance-override",
+            params=_organization_params(organization),
+            json={
+                "override_type": override_type.value,
+                "until": None if until is None else until.isoformat(),
+            },
+        )
+        return _parse_runner(response)
+
+    def clear_instance_override(
+        self, runner_id: UUID, *, organization: str | None
+    ) -> Runner:
+        response = self._client.delete(
+            f"{self._base_url}/v0/runners/{runner_id}/instance-override",
+            params=_organization_params(organization),
+        )
+        return _parse_runner(response)
+
+    def pause_placement(
+        self, runner_id: UUID, *, until: datetime, organization: str | None
+    ) -> Runner:
+        response = self._client.put(
+            f"{self._base_url}/v0/runners/{runner_id}/placement-pause",
+            params=_organization_params(organization),
+            json={"until": until.isoformat()},
+        )
+        return _parse_runner(response)
+
+    def resume_placement(self, runner_id: UUID, *, organization: str | None) -> Runner:
+        response = self._client.delete(
+            f"{self._base_url}/v0/runners/{runner_id}/placement-pause",
+            params=_organization_params(organization),
+        )
+        return _parse_runner(response)
+
+    def create_access_token(
+        self,
+        name: str,
+        scopes: list[Scope],
+        *,
+        expires_at: datetime | None,
+    ) -> CreatedAccessToken:
+        response = self._client.post(
+            f"{self._base_url}/v0/tokens",
+            json={
+                "name": name,
+                "scopes": [scope.value for scope in scopes],
+                "expires_at": (
+                    expires_at.isoformat() if expires_at is not None else None
+                ),
+            },
+        )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return CreatedAccessToken.model_validate(response.json())
+
+    def list_access_tokens(self) -> list[AccessToken]:
+        response = self._client.get(f"{self._base_url}/v0/tokens")
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return _ACCESS_TOKENS_ADAPTER.validate_python(response.json())
+
+    def revoke_access_token(self, token_id: UUID) -> AccessToken:
+        response = self._client.post(f"{self._base_url}/v0/tokens/{token_id}/revoke")
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise AccessTokenNotFoundError(
+                error_detail(response.text) or "access token not found"
+            )
+        if response.is_error:
+            raise ManifoldApiError(response.status_code, response.text)
+        return AccessToken.model_validate(response.json())
+
+
+def _assigned_side_body(response: httpx.Response, run_id: UUID) -> object:
+    """The body of a call on a side assigned to this runner, or why it was refused.
+
+    A 409 is a call the run's state or this runner's standing does not allow,
+    not an API fault. Only the server's `detail` distinguishes them, and the reactions
+    differ: a side that has already reported is finished with, and a side no
+    longer assigned to this runner has its worker torn down.
+    """
+    if response.status_code == HTTPStatus.NOT_FOUND:
+        raise RunNotFoundError(error_detail(response.text) or f"run {run_id} not found")
+    if response.status_code == HTTPStatus.CONFLICT:
+        raise RunSideRefusedError(
+            error_detail(response.text) or f"run {run_id} refused the call"
+        )
+    if response.is_error:
+        raise ManifoldApiError(response.status_code, response.text)
+    return response.json()
+
+
+def _assignment_body(response: httpx.Response) -> object:
+    """The assignment body, or why it is no longer assigned to this runner."""
+    if response.status_code == HTTPStatus.CONFLICT:
+        raise AssignmentNotAssignedToRunnerError(
+            _refusal_message(raw_error_detail(response.text))
+            or "an assignment is not assigned to this runner"
+        )
+    if response.is_error:
+        raise ManifoldApiError(response.status_code, response.text)
+    return response.json()
+
+
+def _heartbeat_refusal(
+    response: httpx.Response, assignment_id: UUID
+) -> AssignmentNotAssignedToRunnerError:
+    """The error for a refused heartbeat, with the run's status where one was sent.
+
+    A server that predates `run_status` sends `detail` as a string. A
+    `run_status` outside `RunStatus` is treated as absent, so the side is
+    handled as reclaimed.
+    """
+    detail = raw_error_detail(response.text)
+    message = (
+        _refusal_message(detail)
+        or f"assignment {assignment_id} is not assigned to this runner"
+    )
+    if not isinstance(detail, dict):
+        return AssignmentNotAssignedToRunnerError(message)
+    try:
+        run_status = RunStatus(detail.get("run_status"))
+    except ValueError:
+        run_status = None
+    return AssignmentNotAssignedToRunnerError(message, run_status=run_status)
+
+
+def _refusal_message(detail: object) -> str | None:
+    """The message of an assignment refusal's `detail`.
+
+    A server that predates the `detail` object sends the message as a string.
+    """
+    if isinstance(detail, dict):
+        detail = detail.get("message")
+    return detail if isinstance(detail, str) else None
+
+
+def _refusal_code(body: str) -> str | None:
+    """The `code` of a task-queue refusal, whose `detail` is an object."""
+    try:
+        payload = json.loads(body)
+    except ValueError:
+        return None
+    detail = payload.get("detail") if isinstance(payload, dict) else None
+    code = detail.get("code") if isinstance(detail, dict) else None
+    return code if isinstance(code, str) else None
+
+
+def _raise_for_register_rejection(response: httpx.Response, slug: str) -> None:
+    """Raise CatalogRegisterError for a refused catalog write.
+
+    The server refuses one of two ways: 409 when another member of the
+    organization owns the slug, 403 when the visibility asked for is the
+    operator organization's to set. Both are the caller's to fix, so neither
+    is an API failure.
+    """
+    if response.status_code not in _REGISTER_REJECTIONS:
+        return
+    raise CatalogRegisterError(
+        error_detail(response.text) or f"'{slug}' could not be registered"
+    )
+
+
+_REGISTER_REJECTIONS = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT})
+
+
+def _page_params(limit: int | None, cursor: str | None) -> dict[str, str | int]:
+    """Query params for a paged list call.
+
+    Unset params are omitted so the server applies its own defaults.
+    """
+    params: dict[str, str | int] = {}
+    if limit is not None:
+        params["limit"] = limit
+    if cursor is not None:
+        params["cursor"] = cursor
+    return params
+
+
+def _parse_runner(response: httpx.Response) -> Runner:
+    """The runner from an override route's response."""
+    if response.is_error:
+        raise ManifoldApiError(response.status_code, response.text)
+    return Runner.model_validate(response.json())
+
+
+def _organization_params(organization: str | None) -> dict[str, str]:
+    """Query params for an optional organization, empty when the caller gave none."""
+    if organization is None:
+        return {}
+    return {"organization": organization}
+
+
+def is_transient_api_failure(exc: BaseException) -> bool:
+    """Whether a call failed in transport or with a 5xx response."""
+    return isinstance(exc, httpx.TransportError) or (
+        isinstance(exc, ManifoldApiError) and exc.status_code >= 500
+    )