benchmark-router-cli 0.1.0__py3-none-any.whl

benchmark_router_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.5
+Name: benchmark-router-cli
+Version: 0.1.0
+Summary: Command-line client for Benchmark Router
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pyyaml>=6
+Requires-Dist: rich>=13
+Requires-Dist: typer>=0.12

benchmark_router_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+brouter_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+brouter_cli/client.py,sha256=DDKCEPha2wKdtHO6NOAZc_yAQvYpAWTXUiH4bX24omg,3699
+brouter_cli/config.py,sha256=IdrZ38hLFYVFF1OvDAORCsyFBxEquENXmFbXX7kdr9o,1345
+brouter_cli/main.py,sha256=9QDtp8IEalr93dcidZeQN0T2LlXWBV5UIIrxBlF81ho,20931
+benchmark_router_cli-0.1.0.dist-info/METADATA,sha256=BHwZqS7hywoahA3N2NhZhdWskM7747jxkEawlJFYxrc,241
+benchmark_router_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+benchmark_router_cli-0.1.0.dist-info/entry_points.txt,sha256=ZcKWWoqvGW9LkeRGfgNTL0QKkKeeYOrWni3XehA7BQU,49
+benchmark_router_cli-0.1.0.dist-info/RECORD,,

benchmark_router_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

benchmark_router_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+brouter = brouter_cli.main:app

brouter_cli/client.py

@@ -0,0 +1,95 @@
+from typing import Any
+
+import httpx
+
+from .config import Settings
+
+
+class ApiError(Exception):
+    def __init__(self, status: int, message: str):
+        super().__init__(f"{status}: {message}")
+        self.status = status
+        self.message = message
+
+
+class Client:
+    def __init__(self, settings: Settings, transport: httpx.BaseTransport | None = None):
+        if not settings.api_key:
+            raise ApiError(401, "no API key; run `brouter login` or set BROUTER_API_KEY")
+        self.api_url = settings.api_url
+        self._http = httpx.Client(
+            base_url=f"{settings.api_url}/api/v1",
+            headers={"Authorization": f"Bearer {settings.api_key}"},
+            timeout=60,
+            transport=transport,
+        )
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        resp = self._http.request(method, path, **kwargs)
+        if resp.status_code >= 400:
+            try:
+                body = resp.json()
+                message = body.get("error") or body.get("message") or resp.text
+            except ValueError:
+                message = resp.text
+            raise ApiError(resp.status_code, str(message))
+        return resp.json()
+
+    def benchmarks(self) -> list[dict]:
+        return self._request("GET", "/benchmarks")["benchmarks"]
+
+    def models(self) -> list[dict]:
+        return self._request("GET", "/models")["models"]
+
+    def estimate(self, body: dict) -> dict:
+        return self._request("POST", "/estimate", json=body)
+
+    def create_run(self, body: dict) -> dict:
+        return self._request("POST", "/runs", json=body)["run"]
+
+    def runs(self) -> list[dict]:
+        return self._request("GET", "/runs")["runs"]
+
+    def run(self, run_id: str) -> dict:
+        return self._request("GET", f"/runs/{run_id}")["run"]
+
+    def download(self, run_id: str) -> list[dict]:
+        return self._request("GET", f"/runs/{run_id}/download")["files"]
+
+    def balance(self) -> dict:
+        return self._request("GET", "/balance")
+
+    def task(self, run_id: str) -> dict:
+        return self._request("GET", f"/runs/{run_id}/task")
+
+    def comparisons(self) -> list[dict]:
+        return self._request("GET", "/comparisons")["comparisons"]
+
+    def estimate_comparison(self, harness_id: str, model_ids: list[str]) -> dict:
+        return self._request("POST", "/comparisons", json={"harness_id": harness_id, "model_ids": model_ids, "dry_run": True})
+
+    def create_comparison(self, harness_id: str, model_ids: list[str]) -> dict:
+        # The server submits one run per benchmark x entrant, so this can take minutes.
+        body = {"harness_id": harness_id, "model_ids": model_ids}
+        return self._request("POST", "/comparisons", json=body, timeout=300)["comparison"]
+
+    def update_comparison(self, comparison_id: str, body: dict) -> dict:
+        return self._request("PATCH", f"/comparisons/{comparison_id}", json=body)["comparison"]
+
+    def harness_requests(self) -> list[dict]:
+        return self._request("GET", "/harness-requests")["requests"]
+
+    def create_harness_request(self, body: dict) -> dict:
+        return self._request("POST", "/harness-requests", json=body)["request"]
+
+    def benchmark_requests(self) -> list[dict]:
+        return self._request("GET", "/benchmark-requests")["requests"]
+
+    def create_benchmark_request(self, body: dict) -> dict:
+        return self._request("POST", "/benchmark-requests", json=body)["request"]
+
+    def checkout(self, amount: float) -> str:
+        return self._request("POST", "/credits/checkout", json={"credits": amount})["url"]
+
+    def fetch(self, url: str) -> httpx.Response:
+        return httpx.get(url, timeout=600, follow_redirects=True)

brouter_cli/config.py

@@ -0,0 +1,50 @@
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_API_URL = "https://benchmarkrouter.newmeasure.ai"
+
+
+def _config_dir() -> Path:
+    return Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
+
+
+def config_path() -> Path:
+    return _config_dir() / "brouter" / "config.json"
+
+
+def _env(name: str) -> str | None:
+    # EVALHUB_* is the pre-rename spelling, still honoured.
+    return os.environ.get(f"BROUTER_{name}") or os.environ.get(f"EVALHUB_{name}")
+
+
+@dataclass
+class Settings:
+    api_url: str
+    api_key: str | None
+
+
+def load_settings() -> Settings:
+    stored: dict = {}
+    path = config_path()
+    legacy = _config_dir() / "evalh" / "config.json"
+    if not path.exists() and legacy.exists():
+        path = legacy
+    if path.exists():
+        stored = json.loads(path.read_text())
+    return Settings(
+        api_url=(_env("API_URL") or stored.get("api_url") or DEFAULT_API_URL).rstrip("/"),
+        api_key=_env("API_KEY") or stored.get("api_key"),
+    )
+
+
+def save_settings(api_key: str, api_url: str | None) -> Path:
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = {"api_key": api_key}
+    if api_url:
+        data["api_url"] = api_url.rstrip("/")
+    path.write_text(json.dumps(data, indent=2))
+    path.chmod(0o600)
+    return path

brouter_cli/main.py

@@ -0,0 +1,523 @@
+import json
+import time
+from pathlib import Path
+from typing import Optional
+
+import typer
+import yaml
+from rich.console import Console
+from rich.table import Table
+
+from .client import ApiError, Client
+from .config import load_settings, save_settings
+
+app = typer.Typer(help="Run public LLM benchmarks on Benchmark Router.", no_args_is_help=True)
+console = Console()
+
+TERMINAL = {"succeeded", "failed"}
+
+
+def credits(amount: Optional[float]) -> str:
+    """The API reports amounts as decimal credits (1 credit = $1)."""
+    return "-" if amount is None else f"{amount:,.4f} credits"
+
+
+def client() -> Client:
+    try:
+        return Client(load_settings())
+    except ApiError as e:
+        console.print(f"[red]{e.message}[/red]")
+        raise typer.Exit(1)
+
+
+def parse_option(raw: str) -> tuple[str, object]:
+    if "=" not in raw:
+        raise typer.BadParameter(f"expected key=value, got {raw!r}")
+    key, value = raw.split("=", 1)
+    return key.strip(), yaml.safe_load(value)
+
+
+def build_request(
+    benchmark: Optional[str],
+    model: Optional[str],
+    reasoning: Optional[str],
+    option: list[str],
+    subset: list[str],
+    config: Optional[Path],
+    harness: Optional[str] = None,
+) -> dict:
+    body: dict = {}
+    if config:
+        body = yaml.safe_load(config.read_text()) or {}
+        if "benchmark" in body:
+            body["benchmark_id"] = body.pop("benchmark")
+        if "model" in body:
+            body["model_id"] = body.pop("model")
+    if benchmark:
+        body["benchmark_id"] = benchmark
+    if model:
+        body["model_id"] = model
+    if reasoning:
+        body["reasoning"] = reasoning
+    if option:
+        body["options"] = {**body.get("options", {}), **dict(parse_option(o) for o in option)}
+    if harness:
+        body["options"] = {**body.get("options", {}), "harness": harness}
+    if subset:
+        body["subsets"] = subset
+    for required in ("benchmark_id", "model_id"):
+        if not body.get(required):
+            raise typer.BadParameter(f"missing {required.removesuffix('_id')} (argument, --model, or config file)")
+    return body
+
+
+def call(fn, *args):
+    try:
+        return fn(*args)
+    except ApiError as e:
+        console.print(f"[red]Error {e.status}:[/red] {e.message}")
+        raise typer.Exit(1)
+
+
+RunArgs = dict(
+    benchmark=typer.Argument(None, help="Benchmark id, e.g. hle"),
+    model=typer.Option(None, "--model", "-m", help="Model id, e.g. anthropic/claude-opus-5-5"),
+    reasoning=typer.Option(None, "--reasoning", "-r", help="Native reasoning level for the model"),
+    option=typer.Option([], "--option", "-o", help="Benchmark option as key=value (repeatable)"),
+    subset=typer.Option([], "--subset", "-s", help="Benchmark-defined subset id (repeatable); default all"),
+    config=typer.Option(None, "--config", "-c", exists=True, help="YAML file with benchmark/model/reasoning/options/subsets"),
+    harness=typer.Option(
+        None, "--harness", "-H", help="Agent harness (see `brouter harnesses BENCHMARK`); default: the benchmark's pinned one"
+    ),
+)
+
+
+@app.command()
+def login(
+    api_key: str = typer.Option(..., prompt=True, hide_input=True, help="API key from the web app"),
+    api_url: Optional[str] = typer.Option(None, help="Override API base URL"),
+):
+    """Save an API key."""
+    path = save_settings(api_key, api_url)
+    console.print(f"Saved to {path}")
+
+
+@app.command()
+def benchmarks(as_json: bool = typer.Option(False, "--json")):
+    """List available benchmarks."""
+    items = call(client().benchmarks)
+    if as_json:
+        print(json.dumps(items, indent=2))
+        return
+    table = Table("id", "name", "version", "subsets")
+    for b in items:
+        table.add_row(b["id"], b["name"], b["version"], ", ".join(s["id"] for s in b["subsets"]))
+    console.print(table)
+
+
+def harness_choices(b: dict) -> Optional[tuple[str, list[tuple[str, str]]]]:
+    """(default label, [(id, label)]) from a benchmark's `harness` option, or None if it has none."""
+    prop = ((b.get("options_schema") or {}).get("properties") or {}).get("harness")
+    if not prop:
+        return None
+    choices = [(str(c["const"]), c.get("title") or str(c["const"])) for c in prop.get("oneOf") or [] if "const" in c]
+    return prop.get("x-unset-label") or "benchmark default", choices
+
+
+@app.command()
+def harnesses(benchmark: str = typer.Argument(..., help="Benchmark id"), as_json: bool = typer.Option(False, "--json")):
+    """List the agent harnesses a benchmark can run (pass one with `--harness`)."""
+    items = call(client().benchmarks)
+    b = next((x for x in items if x["id"] == benchmark), None)
+    if b is None:
+        console.print(f"[red]Unknown benchmark {benchmark!r}[/red]")
+        raise typer.Exit(1)
+    found = harness_choices(b)
+    if as_json:
+        print(json.dumps({"default": found[0] if found else None, "harnesses": [{"id": i, "label": l} for i, l in (found[1] if found else [])]}, indent=2))
+        return
+    if not found:
+        console.print(f"{benchmark} runs its own harness; it has no harness choice.")
+        return
+    default, choices = found
+    console.print(f"Default (no --harness): {default}")
+    table = Table("id", "harness")
+    for i, label in choices:
+        table.add_row(i, label)
+    console.print(table)
+
+
+@app.command()
+def models(as_json: bool = typer.Option(False, "--json")):
+    """List available models and their reasoning levels."""
+    items = call(client().models)
+    if as_json:
+        print(json.dumps(items, indent=2))
+        return
+    table = Table("id", "name", "reasoning levels", "default")
+    for m in items:
+        table.add_row(m["id"], m["display_name"], ", ".join(m.get("reasoning_levels") or []) or "-", m.get("default_reasoning") or "-")
+    console.print(table)
+
+
+@app.command()
+def estimate(
+    benchmark: Optional[str] = RunArgs["benchmark"],
+    model: Optional[str] = RunArgs["model"],
+    reasoning: Optional[str] = RunArgs["reasoning"],
+    option: list[str] = RunArgs["option"],
+    subset: list[str] = RunArgs["subset"],
+    config: Optional[Path] = RunArgs["config"],
+    harness: Optional[str] = RunArgs["harness"],
+):
+    """Show the price of a run before submitting it."""
+    body = build_request(benchmark, model, reasoning, option, subset, config, harness)
+    est = call(client().estimate, body)
+    table = Table("subset", "price")
+    for u in est["units"]:
+        table.add_row(u["subset"], credits(u["price_credits"]))
+    console.print(table)
+    console.print(f"Estimated price: [bold]{credits(est['estimate_credits'])}[/bold]")
+
+
+@app.command()
+def run(
+    benchmark: Optional[str] = RunArgs["benchmark"],
+    model: Optional[str] = RunArgs["model"],
+    reasoning: Optional[str] = RunArgs["reasoning"],
+    option: list[str] = RunArgs["option"],
+    subset: list[str] = RunArgs["subset"],
+    config: Optional[Path] = RunArgs["config"],
+    harness: Optional[str] = RunArgs["harness"],
+    wait: bool = typer.Option(False, "--wait", "-w", help="Wait for the run to finish"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt"),
+):
+    """Submit a run."""
+    body = build_request(benchmark, model, reasoning, option, subset, config, harness)
+    c = client()
+    est = call(c.estimate, body)
+    if not yes:
+        typer.confirm(f"Estimated price {credits(est['estimate_credits'])}. Submit?", abort=True)
+    r = call(c.create_run, body)
+    console.print(f"Run [bold]{r['id']}[/bold] {r['status']}")
+    if wait:
+        r = wait_for(c, r["id"])
+        show_run(r)
+
+
+def wait_for(c: Client, run_id: str, interval: float = 15) -> dict:
+    with console.status("Waiting for run…"):
+        while True:
+            r = call(c.run, run_id)
+            if r["status"] in TERMINAL:
+                return r
+            time.sleep(interval)
+
+
+def show_run(r: dict) -> None:
+    console.print(f"[bold]{r['id']}[/bold]  {r['benchmark_id']} · {r['model_id']}" + (f" @{r['reasoning']}" if r.get("reasoning") else ""))
+    console.print(f"status: {r['status']}   estimate: {credits(r.get('estimate_credits'))}   charged: {credits(r.get('charge_credits'))}")
+    if r.get("subsets"):
+        console.print(f"subsets: {', '.join(r['subsets'])}")
+    if r.get("error"):
+        console.print(f"[red]error:[/red] {r['error']}")
+    if r.get("metrics"):
+        console.print_json(json.dumps(r["metrics"]))
+
+
+@app.command()
+def runs():
+    """List your runs."""
+    items = call(client().runs)
+    table = Table("id", "benchmark", "model", "status", "charged", "created")
+    for r in items:
+        table.add_row(r["id"], r["benchmark_id"], r["model_id"], r["status"], credits(r.get("charge_credits")), r["created_at"])
+    console.print(table)
+
+
+@app.command()
+def status(run_id: str, wait: bool = typer.Option(False, "--wait", "-w")):
+    """Show a run."""
+    c = client()
+    r = wait_for(c, run_id) if wait else call(c.run, run_id)
+    show_run(r)
+
+
+@app.command()
+def download(run_id: str, dest: Path = typer.Option(Path("."), "--dest", "-d", help="Output directory")):
+    """Download a run's results (results.json) and logs (<subset>/logs.tar.gz)."""
+    c = client()
+    files = call(c.download, run_id)
+    root = dest / run_id
+    for f in files:
+        target = root / f["subset"] / f["name"]
+        target.parent.mkdir(parents=True, exist_ok=True)
+        resp = c.fetch(f["url"])
+        resp.raise_for_status()
+        target.write_bytes(resp.content)
+        console.print(f"  {target}")
+    console.print(f"Downloaded {len(files)} files to {root}")
+
+
+@app.command()
+def balance(ledger: bool = typer.Option(False, "--ledger", help="Show ledger entries")):
+    """Show your credit balance."""
+    b = call(client().balance)
+    console.print(f"Balance: [bold]{credits(b['balance_credits'])}[/bold]")
+    if b.get("reserved_credits"):
+        console.print(f"Reserved by pending runs: {credits(b['reserved_credits'])}")
+        console.print(f"Available for new runs: [bold]{credits(b.get('available_credits'))}[/bold]")
+    if ledger:
+        table = Table("date", "kind", "amount", "ref")
+        for e in b["ledger"]:
+            table.add_row(e["created_at"], e["kind"], credits(e["amount_credits"]), e["ref"])
+        console.print(table)
+
+
+# ---------------------------------------------------------------------------- task view
+
+
+def render_blocks(blocks: list[dict]) -> None:
+    for b in blocks:
+        kind = b.get("kind")
+        cut = " [dim](truncated; full text in the run's logs)[/dim]" if b.get("truncated") else ""
+        if kind == "text":
+            console.print(b["text"], markup=False, highlight=False)
+        elif kind == "thinking":
+            console.print("[dim italic]thinking:[/dim italic]")
+            console.print(b["text"], style="dim", markup=False, highlight=False)
+        elif kind == "tool_call":
+            console.print(f"[cyan]→ tool call[/cyan] [bold]{b['name']}[/bold]")
+            console.print(b["input"], markup=False, highlight=False)
+        elif kind == "tool_result":
+            label = "[red]← tool error[/red]" if b.get("error") else "[green]← tool result[/green]"
+            console.print(label + (f" [bold]{b['name']}[/bold]" if b.get("name") else ""))
+            console.print(b["text"], markup=False, highlight=False)
+        elif kind == "image":
+            console.print(f"[dim][image: {b['note']}][/dim]")
+        elif kind == "omitted":
+            console.print(f"[dim]… {b['count']} earlier messages omitted[/dim]")
+        if cut:
+            console.print(cut)
+
+
+def render_messages(messages: list[dict]) -> None:
+    for m in messages:
+        console.rule(f"[bold]{m['role']}[/bold]", align="left", style="dim")
+        render_blocks(m["blocks"])
+
+
+def render_step(s: dict) -> None:
+    head = f"#{s['index']} {s['role']} · {s['model']} · {s['input_tokens']:,} in / {s['output_tokens']:,} out"
+    if s["status"] != "success":
+        head += " · [red]failed[/red]"
+    if s.get("new_conversation"):
+        head += " · new conversation"
+    console.rule(head, align="left")
+    render_messages(s.get("inputs") or [])
+    if s.get("output"):
+        render_messages([s["output"]])
+    if s.get("error"):
+        console.print(f"[red]error:[/red] {s['error']}")
+
+
+@app.command()
+def task(run_id: str, as_json: bool = typer.Option(False, "--json", help="Print the raw task view")):
+    """Show a single-task run's prompt, transcript and grading."""
+    v = call(client().task, run_id)
+    if as_json:
+        print(json.dumps(v, indent=2))
+        return
+    console.print(f"[bold]subset:[/bold] {v['subset'] or '-'}   [bold]status:[/bold] {v['unit_status']}")
+    if v.get("unit_error"):
+        console.print(f"[red]error:[/red] {v['unit_error']}")
+    console.rule("[bold]Task[/bold]")
+    render_messages(v.get("task") or [])
+    if v.get("steps"):
+        console.rule("[bold]Transcript[/bold]")
+        for s in v["steps"]:
+            render_step(s)
+    if v.get("calls_truncated"):
+        console.print("[yellow]Only the first calls are shown; the rest are in the run's logs.[/yellow]")
+    if v.get("judge"):
+        console.rule("[bold]Judge[/bold]")
+        for s in v["judge"]:
+            render_step(s)
+    if v.get("sample"):
+        console.rule("[bold]Grading[/bold]")
+        console.print_json(json.dumps(v["sample"]))
+
+
+# ---------------------------------------------------------------------------- comparisons
+
+comparisons_app = typer.Typer(help="Compare your harness against models.", no_args_is_help=True)
+app.add_typer(comparisons_app, name="comparisons")
+
+ModelsOpt = typer.Option(..., "--model", "-m", help="Model to compare against (repeatable)")
+
+
+def public_url(c: Client, comp: dict) -> Optional[str]:
+    if not comp.get("published_at") or not comp.get("public_slug"):
+        return None
+    return f"{c.api_url}/r/{comp['public_slug']}"
+
+
+@comparisons_app.command("list")
+def comparisons_list(as_json: bool = typer.Option(False, "--json")):
+    """List your comparisons."""
+    c = client()
+    items = call(c.comparisons)
+    if as_json:
+        print(json.dumps(items, indent=2))
+        return
+    table = Table("id", "title", "benchmarks", "models", "public", "created")
+    for x in items:
+        public = public_url(c, x) or "-"
+        if x.get("published_at") and x.get("custom_domain"):
+            public += f"\nhttps://{x['custom_domain']}"
+        table.add_row(x["id"], x["title"], ", ".join(x["benchmark_ids"]), ", ".join(x["model_ids"]), public, x["created_at"])
+    console.print(table)
+
+
+@comparisons_app.command("estimate")
+def comparisons_estimate(harness_id: str = typer.Argument(..., help="Harness id (see `brouter harness list`)"), model: list[str] = ModelsOpt):
+    """Show how many runs a comparison starts and its estimated price."""
+    est = call(client().estimate_comparison, harness_id, model)
+    console.print(f"{est['runs']} runs, estimated price [bold]{credits(est['estimate_credits'])}[/bold]")
+
+
+@comparisons_app.command("create")
+def comparisons_create(
+    harness_id: str = typer.Argument(..., help="Harness id (see `brouter harness list`)"),
+    model: list[str] = ModelsOpt,
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt"),
+):
+    """Start a comparison: one run per benchmark for your harness and each model."""
+    c = client()
+    est = call(c.estimate_comparison, harness_id, model)
+    if not yes:
+        typer.confirm(f"Start {est['runs']} runs, estimated at {credits(est['estimate_credits'])}?", abort=True)
+    with console.status("Submitting runs…"):
+        comp = call(c.create_comparison, harness_id, model)
+    console.print(f"Comparison [bold]{comp['id']}[/bold] started")
+
+
+@comparisons_app.command("publish")
+def comparisons_publish(comparison_id: str):
+    """Publish a comparison as a public report anyone with the link can view."""
+    c = client()
+    comp = call(c.update_comparison, comparison_id, {"published": True})
+    console.print(f"Published: {public_url(c, comp)}")
+    if comp.get("custom_domain"):
+        console.print(f"Also at: https://{comp['custom_domain']}")
+
+
+@comparisons_app.command("unpublish")
+def comparisons_unpublish(comparison_id: str):
+    """Take a comparison's public report down. Publishing again reuses the same link."""
+    call(client().update_comparison, comparison_id, {"published": False})
+    console.print("Unpublished")
+
+
+@comparisons_app.command("domain")
+def comparisons_domain(
+    comparison_id: str,
+    domain: Optional[str] = typer.Argument(None, help="e.g. benchmarks.yourcompany.com"),
+    clear: bool = typer.Option(False, "--clear", help="Remove the custom domain"),
+):
+    """Serve a published comparison's report on your own domain."""
+    if bool(domain) == clear:
+        raise typer.BadParameter("give a domain or --clear")
+    comp = call(client().update_comparison, comparison_id, {"custom_domain": None if clear else domain})
+    console.print(f"Custom domain: {comp['custom_domain']}" if comp.get("custom_domain") else "Custom domain removed")
+
+
+# ---------------------------------------------------------------------------- requests
+
+ACCESS = ("api", "cli", "app")
+
+
+def read_text(value: Optional[str], path: Optional[Path], name: str) -> Optional[str]:
+    if value and path:
+        raise typer.BadParameter(f"give --{name} or --{name}-file, not both")
+    return path.read_text() if path else value
+
+
+harness_app = typer.Typer(help="Bring your own harness: request an adapter and track it.", no_args_is_help=True)
+app.add_typer(harness_app, name="harness")
+
+
+@harness_app.command("list")
+def harness_list(as_json: bool = typer.Option(False, "--json")):
+    """List your harness requests and their status."""
+    items = call(client().harness_requests)
+    if as_json:
+        print(json.dumps(items, indent=2))
+        return
+    table = Table("id", "name", "access", "benchmarks", "status", "created")
+    for r in items:
+        table.add_row(r["id"], r["name"], r["access"], ", ".join(r["benchmark_ids"]), r["status"], r["created_at"])
+    console.print(table)
+
+
+@harness_app.command("request")
+def harness_request(
+    name: str = typer.Option(..., "--name", help="Your harness's name"),
+    access: str = typer.Option(..., "--access", help="How we reach it: api, cli or app"),
+    benchmark: list[str] = typer.Option(..., "--benchmark", "-b", help="Benchmark to run it on (repeatable)"),
+    details: Optional[str] = typer.Option(None, "--details", help="How to call it: endpoint/auth, install command, etc."),
+    details_file: Optional[Path] = typer.Option(None, "--details-file", exists=True, help="Read --details from a file"),
+    docs_url: Optional[str] = typer.Option(None, "--docs-url", help="Link to its docs"),
+):
+    """Request an adapter so your harness can run benchmarks."""
+    if access not in ACCESS:
+        raise typer.BadParameter("access must be api, cli or app")
+    text = read_text(details, details_file, "details")
+    if not text:
+        raise typer.BadParameter("--details or --details-file is required")
+    body = {"name": name, "access": access, "benchmark_ids": benchmark, "details": text, "docs_url": docs_url}
+    r = call(client().create_harness_request, body)
+    console.print(f"Harness request [bold]{r['id']}[/bold] {r['status']}")
+
+
+benchmark_request_app = typer.Typer(help="Ask for a benchmark that isn't on Benchmark Router yet.", no_args_is_help=True)
+app.add_typer(benchmark_request_app, name="benchmark-request")
+
+
+@benchmark_request_app.command("list")
+def benchmark_request_list(as_json: bool = typer.Option(False, "--json")):
+    """List your benchmark requests and their status."""
+    items = call(client().benchmark_requests)
+    if as_json:
+        print(json.dumps(items, indent=2))
+        return
+    table = Table("id", "name", "url", "status", "created")
+    for r in items:
+        table.add_row(r["id"], r["name"], r.get("url") or "-", r["status"], r["created_at"])
+    console.print(table)
+
+
+@benchmark_request_app.command("create")
+def benchmark_request_create(
+    name: str = typer.Argument(..., help="Benchmark name"),
+    url: Optional[str] = typer.Option(None, "--url", help="Paper, repo or leaderboard link"),
+    notes: Optional[str] = typer.Option(None, "--notes", help="Anything else we should know"),
+):
+    """Request a benchmark."""
+    r = call(client().create_benchmark_request, {"name": name, "url": url, "notes": notes})
+    console.print(f"Benchmark request [bold]{r['id']}[/bold] {r['status']}")
+
+
+# ---------------------------------------------------------------------------- credits
+
+
+@app.command()
+def buy(
+    amount: float = typer.Argument(..., help="Credits to buy (1 credit = $1)"),
+    open_browser: bool = typer.Option(True, "--open/--no-open", help="Open the checkout page in your browser"),
+):
+    """Buy credits: creates a Stripe checkout page to pay on."""
+    url = call(client().checkout, amount)
+    console.print(f"Pay here: {url}")
+    if open_browser:
+        typer.launch(url)