manifold-cli 0.0.39__py3-none-any.whl

manifold_cli/commands/runner.py

@@ -0,0 +1,852 @@
+"""Register and manage the runners on this box."""
+
+from datetime import UTC, datetime
+from pathlib import Path
+from shlex import join
+from typing import Annotated, assert_never
+from uuid import UUID
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+from rich.table import Table
+
+from manifold_cli.commands.organization import (
+    ORGANIZATION_CALL_ERRORS,
+    exit_for_call_error,
+)
+from manifold_cli.commands.table import relative_time
+from manifold_cli.dependencies import Dependencies, make_runner_registration_client
+from manifold_cli.models.credentials import RunnerCredential
+from manifold_cli.models.errors import (
+    AccessTokenRejectedError,
+    AmbiguousRunnerHomeError,
+    InvalidAccessTokenError,
+    NoRunnerHomeError,
+    NotLoggedInError,
+    RunnerDeploymentMismatchError,
+)
+from manifold_cli.models.organization import OrganizationNotFoundError
+from manifold_cli.models.runner import (
+    GpuModel,
+    InstanceOverrideType,
+    ListedRunner,
+    Runner,
+    RunnerNameTakenError,
+    RunnerNotFoundError,
+    RunnerProvider,
+    RunnerStatus,
+)
+from manifold_cli.models.runner_home import BoxFacts, RunnerHome
+from manifold_cli.runner.gpu import gpu_facts, gpu_memory_gb
+from manifold_cli.runner.ram import host_ram_gb
+from manifold_cli.services.runner import RunnerService
+from manifold_cli.services.runner_daemon import RunnerDaemonServiceError
+
+ORG_OPTION = typer.Option(
+    "--org", help="Organization identifier; defaults to the operator organization."
+)
+_UNTIL_HELP = (
+    "End time as an ISO 8601 timestamp, such as 2026-11-04T15:00 or "
+    "2026-11-04T15:00+08:00. A timestamp without a time zone is local time."
+)
+
+
+def register(app: typer.Typer, deps: Dependencies) -> None:
+    console = Console()
+    runner_app = typer.Typer(
+        name="runner",
+        help=(
+            "Register runners on this box, list an organization's runners, and "
+            "set their instance overrides and placement pauses."
+        ),
+        no_args_is_help=True,
+        # Only operators run these, so they stay out of a customer's --help.
+        hidden=True,
+    )
+    app.add_typer(runner_app)
+
+    @runner_app.command("register")
+    def register_runner(
+        name: Annotated[str, typer.Argument(help="Name for the runner.")],
+        gpu_model: Annotated[
+            GpuModel,
+            typer.Option(
+                "--gpu-model",
+                help="GPU model of this box. Manifold charges for billed time per "
+                "GPU of this model.",
+            ),
+        ],
+        force: Annotated[
+            bool,
+            typer.Option(
+                "--force",
+                help="Replace an existing runner on this box.",
+            ),
+        ] = False,
+        cache_dir: Annotated[
+            Path | None,
+            typer.Option("--cache-dir", help="Directory this box's workers cache in."),
+        ] = None,
+        advertise_host: Annotated[
+            str | None,
+            typer.Option(
+                "--advertise-host",
+                help="Address this box's policy publishes for its peer to "
+                "dial. Defaults to the address the route towards Manifold uses.",
+            ),
+        ] = None,
+        provider: Annotated[
+            RunnerProvider | None,
+            typer.Option(
+                "--provider",
+                help="Cloud provider of the VM this box runs on.",
+            ),
+        ] = None,
+        is_preemptible: Annotated[
+            bool,
+            typer.Option(
+                "--preemptible",
+                help="The provider may stop this box's VM at any time.",
+            ),
+        ] = False,
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Register a runner and start its persistent system service.
+
+        The command stores the runner token in its home. MANIFOLD_TOKEN takes
+        precedence over the operator's stored WorkOS login. The persistent
+        service runs as the user who registers it. The command returns after
+        systemd enables and starts the service.
+        """
+        access_token = deps.manifold_token
+        runner_service = deps.runner_service
+        registration_client = None
+        try:
+            if access_token is not None:
+                registration_client = make_runner_registration_client(
+                    access_token, deps.manifold_api_url
+                )
+                runner_service = RunnerService(registration_client)
+            _register_runner(
+                name=name,
+                force=force,
+                cache_dir=cache_dir,
+                advertise_host=advertise_host,
+                gpu_model=gpu_model,
+                provider=provider,
+                is_preemptible=is_preemptible,
+                organization=organization,
+                access_token=access_token,
+                runner_service=runner_service,
+                deps=deps,
+                console=console,
+            )
+        except InvalidAccessTokenError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        finally:
+            if registration_client is not None:
+                registration_client.close()
+
+    @runner_app.command()
+    def unregister(
+        runner: Annotated[
+            str | None,
+            typer.Option(
+                "--runner",
+                help="Which home, by name or by id, on a box holding several.",
+            ),
+        ] = None,
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Revoke and remove a runner registered on this box."""
+        try:
+            home = deps.runner_homes.resolve(runner)
+        except (NoRunnerHomeError, AmbiguousRunnerHomeError) as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        stored = home.load()
+        if stored is None:
+            console.print(
+                f"[red]Could not read runner home at {escape(str(home.path))}.[/red]",
+                soft_wrap=True,
+            )
+            raise typer.Exit(1)
+        if stored.credential.api_url.rstrip("/") != deps.manifold_api_url.rstrip("/"):
+            mismatch = RunnerDeploymentMismatchError(
+                stored.credential.api_url, deps.manifold_api_url
+            )
+            console.print(f"[red]{escape(str(mismatch))}[/red]")
+            command = join(
+                (
+                    "env",
+                    "-u",
+                    "MANIFOLD_TOKEN",
+                    f"MANIFOLD_API_URL={stored.credential.api_url}",
+                    "manifold",
+                    "runner",
+                    "unregister",
+                    "--runner",
+                    stored.name,
+                )
+            )
+            console.print(f"Run `{escape(command)}`.", soft_wrap=True)
+            raise typer.Exit(1)
+        try:
+            deps.runner_daemon_service.uninstall(home)
+        except RunnerDaemonServiceError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        try:
+            revoked = deps.runner_service.revoke(
+                stored.runner_id, organization=organization
+            )
+        except RunnerNotFoundError as exc:
+            console.print("[red]Runner service removed, but revoke failed.[/red]")
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            _print_unregister_retry(
+                console, name=stored.name, organization=organization
+            )
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            console.print("[red]Runner service removed, but revoke failed.[/red]")
+            _print_unregister_retry(
+                console, name=stored.name, organization=organization
+            )
+            raise exit_for_call_error(console, exc) from exc
+        home.clear()
+        console.print(
+            f"[green]✓ Runner unregistered[/green] "
+            f"[bold]{revoked.id}[/bold]  {escape(revoked.name)}"
+        )
+
+    @runner_app.command("list")
+    def list_runners(
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """List the active runners, alphabetically."""
+        try:
+            runners = deps.runner_service.list(organization=organization)
+        except OrganizationNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if len(runners) == 0:
+            register_command = _command(
+                "runner",
+                "register",
+                "NAME",
+                "--gpu-model",
+                "GPU_MODEL",
+                *_org_arguments(organization),
+            )
+            console.print(
+                f"There are no runners yet. Register one with: "
+                f"`{escape(register_command)}`"
+            )
+            return
+        active_runners = sorted(
+            (runner for runner in runners if runner.revoked_at is None),
+            key=lambda runner: runner.name.casefold(),
+        )
+        if len(active_runners) == 0:
+            console.print("There are no active runners.")
+            return
+        console.print(_runner_summary(active_runners, console.width))
+
+    @runner_app.command()
+    def show(
+        name: Annotated[str, typer.Argument(help="Runner name.")],
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Show one active runner's details."""
+        runner = _resolve_runner(console, deps.runner_service, name, organization)
+        console.print(_runner_details(runner))
+
+    @runner_app.command("keep-running")
+    def keep_running(
+        name: Annotated[str, typer.Argument(help="Runner name.")],
+        until: Annotated[
+            datetime,
+            typer.Option(help=_UNTIL_HELP, parser=_parse_until, metavar="TIME"),
+        ],
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Have the runner instance service start a managed runner's instance and
+        keep it running until the given time, for maintenance or to add capacity
+        ahead of runs.
+
+        The runner instance service starts the instance if it is stopped and does
+        not stop it before --until, even while the runner is idle or its daemon
+        is not responding. Placement still assigns runs to the runner; add
+        `pause-placement` to keep runs off it during maintenance.
+        """
+        runner = _resolve_runner(console, deps.runner_service, name, organization)
+        try:
+            updated = deps.runner_service.keep_running(
+                runner.id, until=until, organization=organization
+            )
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(_runner_overrides(updated, datetime.now(UTC)))
+
+    @runner_app.command("keep-stopped")
+    def keep_stopped(
+        name: Annotated[str, typer.Argument(help="Runner name.")],
+        until: Annotated[
+            datetime | None,
+            typer.Option(
+                help=f"{_UNTIL_HELP} Omit it to keep the override in effect until "
+                "`clear-overrides`.",
+                parser=_parse_until,
+                metavar="TIME",
+            ),
+        ] = None,
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Stop the runner instance service from starting a managed runner's
+        instance, for example to take a broken VM out of the fleet.
+
+        Setting the override does not stop an instance that is already running,
+        but the service still stops it once it is idle or unresponsive. Add
+        `pause-placement` to keep runs off the runner as well.
+        """
+        runner = _resolve_runner(console, deps.runner_service, name, organization)
+        try:
+            updated = deps.runner_service.keep_stopped(
+                runner.id, until=until, organization=organization
+            )
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(_runner_overrides(updated, datetime.now(UTC)))
+
+    @runner_app.command("pause-placement")
+    def pause_placement(
+        name: Annotated[str, typer.Argument(help="Runner name.")],
+        until: Annotated[
+            datetime,
+            typer.Option(help=_UNTIL_HELP, parser=_parse_until, metavar="TIME"),
+        ],
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Stop placement from assigning runs to a runner until the given time.
+
+        Works on any runner, including one that Bifrost manages manually. A run
+        already on the runner keeps running until it finishes.
+        """
+        runner = _resolve_runner(console, deps.runner_service, name, organization)
+        try:
+            updated = deps.runner_service.pause_placement(
+                runner.id, until=until, organization=organization
+            )
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(_runner_overrides(updated, datetime.now(UTC)))
+
+    @runner_app.command("clear-overrides")
+    def clear_overrides(
+        name: Annotated[str, typer.Argument(help="Runner name.")],
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Clear a runner's instance override and resume placement on it.
+
+        After a `keep-running` override is cleared, the runner instance service
+        stops the instance once it has been idle for the idle timeout, counted
+        from the clear, or once its daemon has not been seen within the liveness
+        window.
+        """
+        runner = _resolve_runner(console, deps.runner_service, name, organization)
+        try:
+            updated = deps.runner_service.clear_overrides(
+                runner.id, organization=organization
+            )
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(_runner_overrides(updated, datetime.now(UTC)))
+
+    @runner_app.command()
+    def revoke(
+        runner_id: Annotated[UUID, typer.Argument(help="Runner id.")],
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Retire a runner's credential."""
+        try:
+            runner = deps.runner_service.revoke(runner_id, organization=organization)
+        except RunnerNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(
+            f"[green]✓ Runner revoked[/green] [bold]{runner.id}[/bold]  "
+            f"{escape(runner.name)}"
+        )
+
+
+def _register_runner(
+    *,
+    name: str,
+    force: bool,
+    cache_dir: Path | None,
+    advertise_host: str | None,
+    gpu_model: GpuModel,
+    provider: RunnerProvider | None,
+    is_preemptible: bool,
+    organization: str | None,
+    access_token: str | None,
+    runner_service: RunnerService,
+    deps: Dependencies,
+    console: Console,
+) -> None:
+    registered = deps.runner_homes.named(name)
+    if registered is not None and not force:
+        # NOTE: a home path is read to be typed at, and Rich would wrap it
+        # on a narrow terminal and put a newline through it. soft_wrap
+        # leaves the line whole and lets the terminal reflow it.
+        console.print(
+            f"[red]A runner is already registered at "
+            f"{escape(str(registered.home.path))}.[/red]",
+            soft_wrap=True,
+        )
+        if access_token is not None:
+            console.print("The existing registration remains.")
+            _print_operator_recovery(
+                console,
+                name=registered.stored.name,
+                gpu_model=gpu_model,
+                organization=organization,
+            )
+        else:
+            console.print(
+                "Register over it with --force, which revokes the runner it holds."
+            )
+        raise typer.Exit(1)
+    if registered is not None:
+        stored = registered.stored.credential.api_url
+        if stored != deps.manifold_api_url:
+            mismatch = RunnerDeploymentMismatchError(stored, deps.manifold_api_url)
+            console.print(f"[red]{escape(str(mismatch))}[/red]")
+            unregister = join(
+                (
+                    "env",
+                    "-u",
+                    "MANIFOLD_TOKEN",
+                    f"MANIFOLD_API_URL={stored}",
+                    "manifold",
+                    "runner",
+                    "unregister",
+                    "--runner",
+                    registered.stored.name,
+                )
+            )
+            retry = _command(
+                "runner",
+                "register",
+                name,
+                "--gpu-model",
+                gpu_model,
+                *_org_arguments(organization),
+            )
+            console.print(
+                "Revoking from here would retire a runner on the other deployment. "
+                f"Run `{escape(unregister)}`, then retry "
+                f"`{escape(retry)}`.",
+                soft_wrap=True,
+            )
+            raise typer.Exit(1)
+        if access_token is not None:
+            console.print(
+                "[red]The manifold runner register command uses MANIFOLD_TOKEN "
+                "only for registration and does not route revoke through that "
+                "client.[/red]"
+            )
+            console.print("The existing registration remains.")
+            _print_operator_recovery(
+                console,
+                name=registered.stored.name,
+                gpu_model=gpu_model,
+                organization=organization,
+            )
+            raise typer.Exit(1)
+    try:
+        # Revoke before minting: the server's name uniqueness covers
+        # unrevoked runners only, so the name stays taken until the one
+        # holding it is retired. The home goes with it, since the runner
+        # minted next keys a directory of its own.
+        if registered is not None:
+            deps.runner_daemon_service.uninstall(registered.home)
+            runner_service.revoke(
+                registered.stored.runner_id, organization=organization
+            )
+            registered.home.clear()
+        facts = gpu_facts()
+        minted = runner_service.mint(
+            name,
+            ram_gb=host_ram_gb(),
+            gpu_memory_gb=gpu_memory_gb(facts.devices),
+            gpu_devices=facts.devices,
+            gpu_model=gpu_model,
+            provider=provider,
+            is_preemptible=is_preemptible,
+            organization=organization,
+        )
+    except RunnerNameTakenError as exc:
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        org_arguments = _org_arguments(organization)
+        list_command = _command("runner", "list", *org_arguments)
+        revoke_command = _command("runner", "revoke", "RUNNER_ID", *org_arguments)
+        retry_command = _command(
+            "runner", "register", name, "--gpu-model", gpu_model, *org_arguments
+        )
+        if access_token is not None:
+            console.print(
+                "The manifold runner register command uses MANIFOLD_TOKEN only "
+                "for registration and does not route revoke through that client. "
+                f"Ask an operator to run `{escape(list_command)}`, substitute the "
+                f"real ID for RUNNER_ID in `{escape(revoke_command)}`, then run "
+                f"`{escape(retry_command)}`.",
+                soft_wrap=True,
+            )
+        else:
+            console.print(
+                "Another box in this or another organization holds that name, or "
+                "its home was removed without revoking it. Run "
+                f"`{escape(list_command)}`, substitute the real ID "
+                f"for RUNNER_ID in `{escape(revoke_command)}`, then run "
+                f"`{escape(retry_command)}`.",
+                soft_wrap=True,
+            )
+        raise typer.Exit(1) from exc
+    except (RunnerNotFoundError, OrganizationNotFoundError) as exc:
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+    except NotLoggedInError as exc:
+        console.print("No registration credential is available.")
+        console.print(
+            "Export MANIFOLD_TOKEN or run `manifold auth login` to store an "
+            "operator credential.",
+            soft_wrap=True,
+        )
+        raise typer.Exit(1) from exc
+    except ORGANIZATION_CALL_ERRORS as exc:
+        raise exit_for_call_error(console, exc) from exc
+    except AccessTokenRejectedError as exc:
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        console.print(
+            "Check that MANIFOLD_TOKEN was created by the deployment configured "
+            "in MANIFOLD_API_URL.",
+            soft_wrap=True,
+        )
+        raise typer.Exit(1) from exc
+    except RunnerDaemonServiceError as exc:
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+    home = deps.runner_homes.for_runner(minted.runner.id)
+    home.save(
+        RunnerHome(
+            name=name,
+            runner_id=minted.runner.id,
+            credential=RunnerCredential(
+                api_url=deps.manifold_api_url, token=minted.token
+            ),
+            box=BoxFacts(
+                gpu_devices=[device.index for device in facts.devices],
+                cache_dir=cache_dir,
+                advertise_host=advertise_host,
+            ),
+        )
+    )
+    try:
+        unit_name = deps.runner_daemon_service.install(home)
+    except RunnerDaemonServiceError as exc:
+        if access_token is not None:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            console.print("The incomplete registration remains.")
+            _print_operator_recovery(
+                console, name=name, gpu_model=gpu_model, organization=organization
+            )
+            raise typer.Exit(1) from exc
+        try:
+            runner_service.revoke(minted.runner.id, organization=organization)
+        except (RunnerNotFoundError, *ORGANIZATION_CALL_ERRORS):
+            retry_command = _command(
+                "runner",
+                "register",
+                name,
+                "--gpu-model",
+                gpu_model,
+                "--force",
+                *_org_arguments(organization),
+            )
+            console.print(
+                "[red]The runner was registered, but its service could not "
+                "start and the registration could not be revoked.[/red]"
+            )
+            console.print(f"Fix the service error, then run `{escape(retry_command)}`.")
+            raise typer.Exit(1) from exc
+        home.clear()
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        console.print("The incomplete registration was revoked.")
+        raise typer.Exit(1) from exc
+    console.print(
+        f"[green]✓ Runner registered[/green] [bold]{minted.runner.id}[/bold]  "
+        f"{escape(minted.runner.name)}"
+    )
+    console.print(f"[dim]home[/dim] {escape(str(home.path))}", soft_wrap=True)
+    console.print(f"[dim]service[/dim] {unit_name}")
+
+
+def _print_operator_recovery(
+    console: Console, *, name: str, gpu_model: GpuModel, organization: str | None
+) -> None:
+    org_arguments = _org_arguments(organization)
+    unregister = _command("runner", "unregister", "--runner", name, *org_arguments)
+    register = _command(
+        "runner", "register", name, "--gpu-model", gpu_model, *org_arguments
+    )
+    console.print("An operator must unregister this runner and register again:")
+    console.print(f"`{escape(unregister)}`")
+    console.print(f"`{escape(register)}`")
+
+
+def _print_unregister_retry(
+    console: Console, *, name: str, organization: str | None
+) -> None:
+    retry = _command(
+        "runner", "unregister", "--runner", name, *_org_arguments(organization)
+    )
+    console.print(f"The runner home remains. Retry with `{escape(retry)}`.")
+
+
+def _command(*arguments: str) -> str:
+    return join(("manifold", *arguments))
+
+
+def _org_arguments(organization: str | None) -> tuple[str, ...]:
+    if organization is None:
+        return ()
+    return ("--org", organization)
+
+
+def _resolve_runner(
+    console: Console,
+    runner_service: RunnerService,
+    name: str,
+    organization: str | None,
+) -> ListedRunner:
+    """The active runner with the given name, or exit after reporting why it
+    could not be found."""
+    try:
+        runners = runner_service.list(organization=organization)
+    except OrganizationNotFoundError as exc:
+        console.print(f"[red]{escape(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+    except ORGANIZATION_CALL_ERRORS as exc:
+        raise exit_for_call_error(console, exc) from exc
+    runner = _find_active_runner(runners, name)
+    if runner is None:
+        console.print(f"[red]Runner '{escape(name)}' not found.[/red]")
+        raise typer.Exit(1)
+    return runner
+
+
+def _parse_until(value: str) -> datetime:
+    try:
+        until = datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise typer.BadParameter(f"'{value}' is not an ISO 8601 timestamp") from exc
+    # `astimezone` converts a naive timestamp from local time.
+    return until.astimezone(UTC)
+
+
+def _runner_overrides(runner: Runner, now: datetime) -> Table:
+    table = Table(box=None, pad_edge=False, show_header=False)
+    table.add_column(style="dim", no_wrap=True)
+    table.add_column(overflow="fold")
+    table.add_row("name", escape(runner.name))
+    table.add_row("instance", _instance_override(runner, now))
+    table.add_row("placement", _placement(runner, now))
+    return table
+
+
+def _runner_summary(runners: list[ListedRunner], width: int) -> Table:
+    if width < 80:
+        return _narrow_runner_summary(runners)
+
+    table = Table(box=None, pad_edge=False)
+    table.add_column("Name", overflow="fold")
+    table.add_column("Status", overflow="fold")
+    table.add_column("RAM", overflow="fold")
+    table.add_column("GPU", overflow="fold")
+    table.add_column("Overrides", overflow="fold")
+    table.add_column("Preemptible", overflow="fold")
+    now = datetime.now(UTC)
+    for runner in runners:
+        table.add_row(
+            f"[bold]{escape(runner.name)}[/bold]",
+            _status(runner),
+            _ram(runner),
+            _gpu_summary(runner),
+            _overrides_summary(runner, now),
+            _preemptible(runner),
+        )
+    return table
+
+
+def _narrow_runner_summary(runners: list[ListedRunner]) -> Table:
+    now = datetime.now(UTC)
+    table = Table(box=None, pad_edge=False, show_header=False)
+    table.add_column(style="dim", no_wrap=True)
+    table.add_column(overflow="fold")
+    for runner in runners:
+        table.add_row("name", f"[bold]{escape(runner.name)}[/bold]")
+        table.add_row("status", _status(runner))
+        table.add_row("ram", _ram(runner))
+        table.add_row("gpu", _gpu_summary(runner))
+        table.add_row("overrides", _overrides_summary(runner, now))
+        table.add_row("preemptible", _preemptible(runner), end_section=True)
+    return table
+
+
+def _find_active_runner(runners: list[ListedRunner], name: str) -> ListedRunner | None:
+    return next(
+        (
+            runner
+            for runner in runners
+            if runner.name == name and runner.revoked_at is None
+        ),
+        None,
+    )
+
+
+def _runner_details(runner: ListedRunner) -> Table:
+    table = Table(box=None, pad_edge=False, show_header=False)
+    table.add_column(style="dim", no_wrap=True)
+    table.add_column(overflow="fold")
+    table.add_row("name", escape(runner.name))
+    table.add_row("runner id", str(runner.id))
+    table.add_row("created", f"{runner.created_at:%Y-%m-%d %H:%M}")
+    table.add_row("status", _status(runner))
+    now = datetime.now(UTC)
+    table.add_row("last seen", _last_seen_details(runner, now))
+    table.add_row("ram", _ram(runner))
+    table.add_row("gpus", _gpus(runner))
+    table.add_row("preemptible", _preemptible(runner))
+    table.add_row("instance", _instance_override(runner, now))
+    table.add_row("placement", _placement(runner, now))
+    return table
+
+
+def _status(runner: ListedRunner) -> str:
+    if not isinstance(runner.status, RunnerStatus):
+        return escape(runner.status)
+    if runner.status is RunnerStatus.IDLE:
+        return "[green]idle[/]"
+    elif runner.status is RunnerStatus.BUSY:
+        return "[cyan]busy[/]"
+    elif runner.status is RunnerStatus.STARTING:
+        return "[cyan]starting[/]"
+    elif runner.status is RunnerStatus.NEVER_CONTACTED:
+        return "[yellow]never contacted[/]"
+    elif runner.status is RunnerStatus.OFFLINE:
+        return "[red]offline[/]"
+    elif runner.status is RunnerStatus.REVOKED:
+        return "[red]revoked[/]"
+    else:
+        assert_never(runner.status)
+
+
+def _overrides_summary(runner: Runner, now: datetime) -> str:
+    """The overrides in effect, without their end times. `show` prints the end
+    times."""
+    overrides = []
+    instance_override = runner.instance_override
+    if instance_override is not None and instance_override.is_in_effect(now):
+        overrides.append(_override_type(instance_override.override_type))
+    if runner.is_placement_paused(now):
+        overrides.append("placement paused")
+    if len(overrides) == 0:
+        return "none"
+    return ", ".join(overrides)
+
+
+def _instance_override(runner: Runner, now: datetime) -> str:
+    instance_override = runner.instance_override
+    if instance_override is None or not instance_override.is_in_effect(now):
+        return "none"
+    override_type = _override_type(instance_override.override_type)
+    if instance_override.until is None:
+        return f"{override_type} until cleared"
+    return f"{override_type} until {_local_time(instance_override.until)}"
+
+
+def _placement(runner: Runner, now: datetime) -> str:
+    if runner.placement_paused_until is None or not runner.is_placement_paused(now):
+        return "open"
+    return f"paused until {_local_time(runner.placement_paused_until)}"
+
+
+def _override_type(override_type: InstanceOverrideType | str) -> str:
+    if not isinstance(override_type, InstanceOverrideType):
+        return escape(override_type)
+    if override_type is InstanceOverrideType.KEEP_RUNNING:
+        return "keep running"
+    elif override_type is InstanceOverrideType.KEEP_STOPPED:
+        return "keep stopped"
+    else:
+        assert_never(override_type)
+
+
+def _local_time(instant: datetime) -> str:
+    return f"{instant.astimezone():%Y-%m-%d %H:%M %Z}"
+
+
+def _last_seen_relative(runner: Runner, now: datetime) -> str:
+    if runner.last_seen_at is None:
+        return "never"
+    # A report stamped ahead of this clock is skew, not a sighting yet to come.
+    return relative_time(min(runner.last_seen_at, now), now)
+
+
+def _last_seen_details(runner: Runner, now: datetime) -> str:
+    if runner.last_seen_at is None:
+        return "never"
+    relative = _last_seen_relative(runner, now)
+    exact = runner.last_seen_at.astimezone(UTC).isoformat().replace("+00:00", "Z")
+    return f"{relative} ({exact})"
+
+
+def _ram(runner: Runner) -> str:
+    if runner.ram_gb is None:
+        return "unknown"
+    return f"{runner.ram_gb} GB"
+
+
+def _gpus(runner: Runner) -> str:
+    if runner.gpu_device_count == 0:
+        return "none"
+    return ", ".join(
+        f"{device.index}: {escape(device.name)} ({device.memory_gb} GB)"
+        for device in runner.gpu_devices
+    )
+
+
+def _gpu_summary(runner: Runner) -> str:
+    if runner.gpu_device_count == 0:
+        return "none"
+    if len(runner.gpu_devices) == 0:
+        return str(runner.gpu_device_count)
+    first = runner.gpu_devices[0]
+    if any(
+        device.name != first.name or device.memory_gb != first.memory_gb
+        for device in runner.gpu_devices[1:]
+    ):
+        return f"{runner.gpu_device_count} GPUs"
+    count = "" if runner.gpu_device_count == 1 else f"{runner.gpu_device_count} x "
+    return f"{count}{escape(first.name)} ({first.memory_gb} GB)"
+
+
+def _preemptible(runner: Runner) -> str:
+    return "yes" if runner.is_preemptible else "no"