benchkit-cli 0.1.0__py3-none-any.whl

benchkit/cli.py

@@ -0,0 +1,1000 @@
+"""BenchKit entry point.
+
+Running `benchkit` opens the full-screen terminal app. `--headless` keeps a
+scriptable path that prints to stdout instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from dotenv import load_dotenv
+from rich import box
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from benchkit import catalog
+from benchkit.benchmarks import DESCRIPTIONS, REGISTRY
+from benchkit.client import InferenceClient
+from benchkit.engine import (
+    MAX_REPAIR_ATTEMPTS,
+    JobSpec,
+    SliceError,
+    expand_jobs,
+    parse_slice,
+    route_decision_jobs,
+    slice_task_count,
+    task_count,
+)
+from benchkit.history import serve_history
+from benchkit.jobs import job_matrix
+from benchkit.metrics import (
+    aggregate_tok_s,
+    decision_speed,
+    effective_concurrency,
+    is_decision,
+    latency_text,
+    stream_tok_s,
+)
+from benchkit.perf import DEFAULT_DEPTHS, PerfConfig, parse_depths, run_profile
+from benchkit.perf_report import save_profile
+from benchkit.perturbations import (
+    PERTURBATIONS,
+    perturbations_for,
+    unsupported_benchmarks,
+)
+from benchkit.report import save
+from benchkit.runner import run
+from benchkit.tui import run_tui
+
+console = Console(highlight=False)
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="benchkit",
+        description="Benchmark local LLMs with real evaluation suites.",
+    )
+    parser.add_argument(
+        "command",
+        nargs="?",
+        choices=("perf", "history", "render-check", "datasets"),
+        help=(
+            "Run a performance profile, browse historical results, check "
+            "that headless rendering works on this machine, or list and "
+            "install benchmark datasets"
+        ),
+    )
+    parser.add_argument(
+        "perf_model",
+        nargs="?",
+        metavar="MODEL",
+        help=(
+            "Model name for `benchkit perf MODEL`; `install` for "
+            "`benchkit datasets install [BENCHMARK ...]`"
+        ),
+    )
+    parser.add_argument(
+        "targets",
+        nargs="*",
+        metavar="BENCHMARK",
+        help="Datasets: benchmarks to install (default: every missing one)",
+    )
+    parser.add_argument("--host", help="Inference host, overrides BENCHKIT_HOST")
+    parser.add_argument(
+        "--demo",
+        action="store_true",
+        help="Run against a built-in offline model server (no inference server needed)",
+    )
+    parser.add_argument(
+        "--headless",
+        action="store_true",
+        help="Skip the TUI and run from flags, printing progress to stdout",
+    )
+    parser.add_argument(
+        "--models",
+        default="",
+        help="Headless: comma separated model names, or 'all'",
+    )
+    parser.add_argument(
+        "--benchmarks",
+        default="",
+        help=(
+            "Headless: comma separated benchmarks, e.g. humaneval:20,gsm8k:-50 "
+            "or aider-polyglot:cpp:5"
+        ),
+    )
+    parser.add_argument(
+        "--perturbation",
+        choices=PERTURBATIONS,
+        help=("Headless: add a paired perturbed run alongside each clean baseline"),
+    )
+    parser.add_argument(
+        "--perturbation-seed",
+        type=int,
+        default=42,
+        metavar="SEED",
+        help="Headless: deterministic perturbation seed (default: 42)",
+    )
+    parser.add_argument(
+        "--harness",
+        choices=("direct", "pi", "both"),
+        default="direct",
+        help=(
+            "Execution harness: raw API, latest stock Pi in Docker, or paired "
+            "direct + Pi runs (default: direct)"
+        ),
+    )
+    parser.add_argument(
+        "--decision-models",
+        default="",
+        metavar="MODELS",
+        help=(
+            "Comma-separated models to run as decision models over "
+            "/v1/systemone, for servers or proxies that do not advertise them "
+            "(llama.cpp's own listing is detected automatically)"
+        ),
+    )
+    parser.add_argument(
+        "--repair-attempts",
+        type=int,
+        choices=range(MAX_REPAIR_ATTEMPTS + 1),
+        default=0,
+        metavar=f"0-{MAX_REPAIR_ATTEMPTS}",
+        help=(
+            "Verifier-feedback repair turns after an incorrect first answer; "
+            "works with direct and Pi harnesses (default: 0; max: "
+            f"{MAX_REPAIR_ATTEMPTS})"
+        ),
+    )
+    parser.add_argument(
+        "--force-unload",
+        action="store_true",
+        help=(
+            "Headless: evict each model from VRAM between models "
+            "(llama.cpp POST /models/unload; Ollama keep_alive=0)"
+        ),
+    )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="Print the available benchmarks and exit",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Headless: print every prompt and response",
+    )
+    parser.add_argument(
+        "--depths",
+        default=",".join(
+            "minimal" if value == 0 else f"{value // 1024}k" for value in DEFAULT_DEPTHS
+        ),
+        help="Perf: comma-separated context depths (default: minimal,4k,16k,32k,64k)",
+    )
+    parser.add_argument(
+        "--gen",
+        type=int,
+        default=128,
+        metavar="TOKENS",
+        help="Perf: generated tokens per measured run (default: 128)",
+    )
+    parser.add_argument(
+        "--reps",
+        type=int,
+        default=5,
+        help="Perf: measured repetitions per depth (default: 5)",
+    )
+    parser.add_argument(
+        "--warmups",
+        type=int,
+        default=1,
+        help="Perf: discarded warmups per depth (default: 1)",
+    )
+    parser.add_argument(
+        "--config-note",
+        default="",
+        help="Perf: optional hardware or server configuration note",
+    )
+    parser.add_argument(
+        "--results-dir",
+        action="append",
+        default=None,
+        metavar="PATH",
+        help=(
+            "History: results directory to scan; repeat to combine archives "
+            "(default: ./results)"
+        ),
+    )
+    parser.add_argument(
+        "--no-open",
+        action="store_true",
+        help="History: print the local URL without opening a browser",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=0,
+        metavar="PORT",
+        help="History: local dashboard port; 0 chooses a free port (default: 0)",
+    )
+    return parser.parse_args(argv)
+
+
+def _fmt_time(seconds: float) -> str:
+    seconds = round(seconds)
+    if seconds >= 60:
+        return f"{seconds // 60}m {seconds % 60}s"
+    return f"{seconds}s"
+
+
+def _score_text(score: float) -> Text:
+    if score >= 80:
+        style = "bold green"
+    elif score < 50:
+        style = "bold red"
+    else:
+        style = "bold"
+    return Text(f"{score:.1f}%", style=style)
+
+
+def _outcome_counts(result: dict) -> tuple[int, int, int]:
+    """Return pass, ordinary failure and error-task counts for a result."""
+    tasks = result.get("tasks", [])
+    failed = sum(not task.get("passed") and not task.get("error") for task in tasks)
+    errors = sum(not task.get("passed") and bool(task.get("error")) for task in tasks)
+    return int(result["passed"]), failed, errors
+
+
+def _list_benchmarks() -> None:
+    table = Table(box=box.MINIMAL, border_style="dim", header_style="bold")
+    table.add_column("Benchmark")
+    table.add_column("Tasks", justify="right")
+    table.add_column("Description")
+    table.add_column("Perturbations")
+    table.add_column("Notes")
+    for key in REGISTRY:
+        try:
+            declared_label = getattr(REGISTRY[key], "list_task_count", "")
+            count = declared_label or f"{task_count(key):,}"
+        except Exception:
+            count = "?"
+        note = getattr(REGISTRY[key], "list_note", "")
+        table.add_row(
+            key,
+            count,
+            DESCRIPTIONS.get(key, ""),
+            f"[cyan]{' '.join(perturbations_for(key))}[/cyan]",
+            f"[yellow]{note}[/yellow]" if note else "",
+        )
+    console.print(table)
+
+
+def _datasets_table() -> None:
+    table = Table(box=box.MINIMAL, border_style="dim", header_style="bold")
+    table.add_column("Benchmark")
+    table.add_column("Status")
+    table.add_column("Size", justify="right")
+    table.add_column("Source")
+    labels = {
+        "bundled": "[dim]bundled[/dim]",
+        "installed": "[green]✓ installed[/green]",
+        "download": "[yellow]✗ missing[/yellow]",
+        "manual": "[red]✗ manual setup[/red]",
+    }
+    entries = catalog.statuses()
+    for entry in entries:
+        state = (
+            entry.kind
+            if entry.kind == "bundled" or not entry.installed
+            else "installed"
+        )
+        source = entry.summary
+        if entry.kind == "manual" and not entry.installed and entry.hint:
+            source += f"\n[dim]{entry.hint}[/dim]"
+        table.add_row(
+            entry.benchmark,
+            labels[state],
+            catalog.size_label(entry),
+            source,
+        )
+    console.print(table)
+    missing = catalog.missing()
+    if missing:
+        console.print(
+            f"[yellow]{len(missing)} missing:[/yellow] {', '.join(missing)} "
+            "[dim]· run 'benchkit datasets install' to fetch them all[/dim]"
+        )
+    else:
+        console.print("[green]Every downloadable dataset is installed.[/green]")
+
+
+def _install_datasets(keys: list[str]) -> bool:
+    """Install each benchmark's data with a progress line. True if all worked."""
+    from rich.progress import (
+        BarColumn,
+        DownloadColumn,
+        Progress,
+        SpinnerColumn,
+        TextColumn,
+    )
+
+    ok = True
+    for key in keys:
+        entry = catalog.status(key)
+        if entry.installed:
+            note = "bundled" if entry.kind == "bundled" else "already installed"
+            console.print(f"[dim]{key}: {note}[/dim]")
+            continue
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("{task.description}"),
+            BarColumn(),
+            DownloadColumn(),
+            console=console,
+            transient=True,
+        ) as progress:
+            task = progress.add_task(f"Installing {key}", total=entry.size)
+            try:
+                catalog.install(
+                    key,
+                    lambda done, total, task=task: progress.update(
+                        task, completed=done, total=total
+                    ),
+                )
+            except Exception as exc:
+                ok = False
+                progress.stop()
+                console.print(f"[red]✗ {key}:[/red] {exc}")
+                continue
+        console.print(f"[green]✓ {key}[/green] [dim]{entry.location}[/dim]")
+    return ok
+
+
+def _datasets(args: argparse.Namespace) -> None:
+    if args.perf_model is None:
+        _datasets_table()
+        return
+    if args.perf_model != "install":
+        console.print(
+            f"[red]Unknown datasets action:[/red] {args.perf_model} "
+            "[dim]· use 'benchkit datasets' or 'benchkit datasets install'[/dim]"
+        )
+        sys.exit(1)
+    unknown = [key for key in args.targets if key not in REGISTRY]
+    if unknown:
+        console.print(f"[red]Unknown benchmark(s):[/red] {', '.join(unknown)}")
+        sys.exit(1)
+    keys = args.targets or catalog.missing()
+    if not keys:
+        console.print("[green]Every downloadable dataset is already installed.[/green]")
+        return
+    if not _install_datasets(keys):
+        sys.exit(1)
+
+
+def _render_check() -> None:
+    """Report whether this machine can run the suites that draw pictures.
+
+    Every treejs-arena task depends on two things BenchKit does not control:
+    a headless browser that can produce a WebGL canvas, and a route to the
+    module CDN the generated page imports from. This checks both, so a broken
+    server is diagnosed in one command instead of ten scored-zero tasks.
+    mc-arena needs the same browser, but only for its screenshots: its scores
+    come from the block list, and it needs Docker rather than the CDN.
+    """
+    from benchkit.browser import allowed_hosts, probe_environment
+
+    ok, detail = probe_environment(force=True)
+    if ok:
+        console.print(f"[green]Headless rendering works.[/green] [dim]{detail}[/dim]")
+    else:
+        console.print(f"[red]Headless rendering is unavailable.[/red] {detail}")
+
+    hosts = allowed_hosts()
+    if hosts:
+        console.print(
+            "[dim]Generated pages may fetch modules from:[/dim] " + ", ".join(hosts)
+        )
+        console.print(
+            "[dim]The machine needs outbound HTTPS to at least one of them; "
+            "otherwise every scene fails on a blocked import.[/dim]"
+        )
+    else:
+        console.print(
+            "[yellow]Network access is disabled "
+            "(BENCHKIT_RENDER_OFFLINE).[/yellow] [dim]Pages that import "
+            "three.js from a CDN cannot render.[/dim]"
+        )
+    console.print(
+        "[dim]mc-arena additionally needs Docker, and uses the browser only "
+        "for screenshots; its builds are scored either way.[/dim]"
+    )
+    if not ok:
+        sys.exit(1)
+
+
+def _client(args: argparse.Namespace):
+    if args.demo:
+        from benchkit.demo import DemoClient
+
+        return DemoClient()
+
+    client = InferenceClient.from_env()
+    if args.host:
+        client.host = args.host.rstrip("/")
+    return client
+
+
+def _headless_jobs(args: argparse.Namespace, available: list[str]) -> list[JobSpec]:
+    if args.models.strip().lower() in {"all", "*"}:
+        models = available
+    else:
+        models = [name.strip() for name in args.models.split(",") if name.strip()]
+
+    unknown = [name for name in models if name not in available]
+    if unknown:
+        console.print(f"[red]Unknown model(s):[/red] {', '.join(unknown)}")
+        console.print(f"[dim]Available: {', '.join(available)}[/dim]")
+        sys.exit(1)
+    if not models:
+        console.print("[red]--models is required with --headless[/red]")
+        sys.exit(1)
+
+    requested = [part for part in args.benchmarks.split(",") if part.strip()]
+
+    specs: list[tuple[str, str | None, str | None]] = []
+    for part in requested:
+        part = part.strip()
+        if not part:
+            continue
+        pieces = [piece.strip() for piece in part.split(":")]
+        key = pieces[0]
+        if key not in REGISTRY:
+            console.print(f"[red]Unknown benchmark:[/red] {key}")
+            console.print(f"[dim]Available: {', '.join(REGISTRY)}[/dim]")
+            sys.exit(1)
+        if len(pieces) > 3 or any(not piece for piece in pieces[1:]):
+            console.print(f"[red]Invalid benchmark selector:[/red] {part}")
+            sys.exit(1)
+
+        bench = REGISTRY[key]()
+        resolve_variant = getattr(bench, "resolve_variant", None)
+        variant = None
+        slice_spec = None
+        if len(pieces) == 2:
+            candidate = (
+                resolve_variant(pieces[1]) if callable(resolve_variant) else None
+            )
+            if candidate is not None:
+                variant = str(candidate)
+            else:
+                slice_spec = pieces[1]
+        elif len(pieces) == 3:
+            candidate = (
+                resolve_variant(pieces[1]) if callable(resolve_variant) else None
+            )
+            if candidate is None:
+                console.print(f"[red]Unknown {key} variant:[/red] {pieces[1]}")
+                sys.exit(1)
+            variant = str(candidate)
+            slice_spec = pieces[2]
+        try:
+            variant_count = getattr(bench, "variant_task_count", None)
+            total = (
+                int(variant_count(variant))
+                if variant is not None and callable(variant_count)
+                else slice_task_count(key)
+            )
+        except Exception as exc:
+            console.print(f"[red]Could not load {key}:[/red] {exc}")
+            sys.exit(1)
+        try:
+            parse_slice(slice_spec, total)
+        except SliceError as exc:
+            console.print(f"[red]{exc}[/red]")
+            sys.exit(1)
+        specs.append((key, slice_spec, variant))
+
+    if not specs:
+        console.print("[red]--benchmarks is required with --headless[/red]")
+        sys.exit(1)
+
+    if args.perturbation:
+        unsupported = unsupported_benchmarks(
+            args.perturbation, [key for key, _, _ in specs]
+        )
+        if unsupported:
+            console.print(
+                f"[red]{args.perturbation} is unsupported for:[/red] "
+                + ", ".join(unsupported)
+            )
+            supported = [
+                key for key in REGISTRY if args.perturbation in perturbations_for(key)
+            ]
+            console.print(f"[dim]Supported: {', '.join(supported)}[/dim]")
+            sys.exit(1)
+
+    harnesses = ("direct", "pi") if args.harness == "both" else (args.harness,)
+    return job_matrix(
+        models,
+        specs,
+        harnesses,
+        repair_attempts=args.repair_attempts,
+        perturbation=args.perturbation,
+        perturbation_seed=args.perturbation_seed,
+    )
+
+
+def _headless(args: argparse.Namespace) -> None:
+    if args.demo and args.decision_models.strip():
+        console.print("[red]Decision models are unavailable in demo mode.[/red]")
+        sys.exit(1)
+    if args.demo and args.harness != "direct":
+        console.print("[red]Pi harness is unavailable in demo mode.[/red]")
+        sys.exit(1)
+    try:
+        client = _client(args)
+    except ValueError as exc:
+        console.print(f"[red]Configuration error:[/red] {exc}")
+        sys.exit(1)
+
+    console.print(f"[bold]BenchKit[/bold] [dim]{client.host}[/dim]")
+
+    try:
+        models = client.list_models()
+    except Exception as exc:
+        console.print(f"[red]Connection failed:[/red] {exc}")
+        sys.exit(1)
+
+    selected = _headless_jobs(args, [model["name"] for model in models])
+    forced = {name.strip() for name in args.decision_models.split(",") if name.strip()}
+    stray = sorted(forced - {job.model for job in selected})
+    if stray:
+        console.print(
+            "[red]--decision-models names models that are not in --models:[/red] "
+            + ", ".join(stray)
+        )
+        sys.exit(1)
+    try:
+        jobs = route_decision_jobs(selected, client, forced)
+    except ValueError as exc:
+        console.print(f"[red]{exc}[/red]")
+        sys.exit(1)
+    jobs = expand_jobs(jobs, client)
+    if not jobs:
+        console.print(
+            "[red]No supported RULER context bucket fits the selected model(s).[/red]"
+        )
+        sys.exit(1)
+    # Fetch what the run needs up front, with a progress bar, rather than
+    # stalling silently inside the first job's task loader.
+    missing = catalog.missing(dict.fromkeys(job.benchmark for job in jobs))
+    if missing:
+        console.print(f"[dim]Fetching datasets:[/dim] {', '.join(missing)}")
+        # A partial install (RULER without its essay corpus) can still serve
+        # some slices, so leave the verdict to the task loader.
+        if not _install_datasets(missing):
+            console.print(
+                "[yellow]Continuing; tasks that need the missing data will "
+                "fail when they load.[/yellow]"
+            )
+    if args.demo:
+        perturbations = sorted(
+            {
+                (job.perturbation, job.perturbation_seed)
+                for job in jobs
+                if job.perturbation
+            }
+        )
+        client.prime(
+            sorted({job.benchmark for job in jobs}),
+            perturbations=perturbations,
+        )
+
+    results, failure = run(
+        client,
+        jobs,
+        console,
+        args.verbose,
+        force_unload=args.force_unload,
+    )
+    if not results:
+        console.print("[yellow]No results.[/yellow]")
+        sys.exit(1 if failure else 0)
+
+    table = Table(
+        box=box.MINIMAL,
+        border_style="dim",
+        header_style="bold",
+        pad_edge=False,
+        padding=(0, 1),
+    )
+    # Fold long model and benchmark names instead of ellipsizing them: the
+    # summary is the part of a headless run people copy into a report.
+    table.add_column("Model", overflow="fold")
+    table.add_column("Harness")
+    table.add_column("Benchmark", overflow="fold")
+    table.add_column("Score", justify="right")
+    has_harness_pairs = any(
+        result.get("harness_score_delta_pp") is not None for result in results
+    )
+    if has_harness_pairs:
+        table.add_column("Harness score Δ", justify="right")
+        table.add_column("Loop-kill Δ", justify="right")
+    has_repairs = any(result.get("repair_attempts", 0) for result in results)
+    if has_repairs:
+        table.add_column("Repair Δ", justify="right")
+        table.add_column("Fixed", justify="right")
+    has_perturbations = any(result.get("perturbation") for result in results)
+    if has_perturbations:
+        table.add_column("Delta", justify="right")
+        table.add_column("Regress", justify="right")
+    table.add_column("Pass/Scored", justify="right")
+    table.add_column("Fail/Loop/TO/Len/HE", justify="right")
+    has_parallel = any(result.get("concurrency", 1) > 1 for result in results)
+    # Decision rows are measured in dec/s, so a table that mixes them with
+    # generating rows names the unit in every cell instead of the header.
+    has_decisions = any(is_decision(result) for result in results)
+    unit = " tok/s" if has_decisions else ""
+    if has_parallel:
+        table.add_column("Parallel", justify="right", style="dim")
+        table.add_column("Agg speed" if has_decisions else "Agg tok/s", justify="right")
+        table.add_column("Stream tok/s", justify="right", style="dim")
+        table.add_column("Eff", justify="right", style="dim")
+    else:
+        table.add_column("Speed" if has_decisions else "Tok/s", justify="right")
+    table.add_column("Avg Time", justify="right", style="dim")
+    table.add_column("Wall", justify="right", style="dim")
+
+    for result in results:
+        passed, failed, errors = _outcome_counts(result)
+        parallel = result.get("concurrency", 1) > 1
+        row = [
+            result["model"],
+            result.get("harness_label", "Direct"),
+            result.get("benchmark_label", result["benchmark"]),
+            _score_text(result["score"]),
+        ]
+        if has_harness_pairs:
+            harness_delta = result.get("harness_score_delta_pp")
+            row.append(
+                f"{float(harness_delta):+.1f}pp" if harness_delta is not None else "—"
+            )
+            loop_delta = result.get("loop_kill_delta_pp")
+            row.append(f"{float(loop_delta):+.1f}pp" if loop_delta is not None else "—")
+        if has_repairs:
+            if result.get("repair_attempts", 0):
+                row.extend(
+                    [
+                        f"{float(result.get('repair_delta_pp', 0)):+.1f}pp",
+                        (
+                            f"{result.get('repair_successes', 0)}/"
+                            f"{result.get('repair_attempted', 0)}"
+                        ),
+                    ]
+                )
+            else:
+                row.extend(["—", "—"])
+        if has_perturbations:
+            delta = result.get("score_delta_pp")
+            row.extend(
+                [
+                    f"{float(delta):+.1f}pp" if delta is not None else "—",
+                    (
+                        f"{result.get('regressions', 0)}/"
+                        f"{result.get('paired_total', 0)}"
+                        if result.get("perturbation")
+                        else "—"
+                    ),
+                ]
+            )
+        row.extend(
+            [
+                f"{passed}/{result.get('scored_total', result['total'])}",
+                f"{result.get('failures', failed)}/{result.get('loop_kills', 0)}/"
+                f"{result.get('timeouts', 0)}/"
+                f"{result.get('length_exceeded', 0)}/"
+                f"{result.get('harness_errors', errors)}",
+            ]
+        )
+        decisions = is_decision(result)
+        if has_parallel:
+            row.extend(
+                [
+                    str(result.get("concurrency", 1)),
+                    decision_speed(result)[0]
+                    if decisions
+                    else f"{aggregate_tok_s(result):.1f}{unit}"
+                    if parallel
+                    else "—",
+                    "—" if decisions else f"{stream_tok_s(result):.1f}",
+                    f"{effective_concurrency(result):.2f}x" if parallel else "—",
+                ]
+            )
+        else:
+            row.append(
+                decision_speed(result)[0]
+                if decisions
+                else f"{stream_tok_s(result):.1f}{unit}"
+            )
+        row.extend([latency_text(result), _fmt_time(result["total_time"])])
+        table.add_row(
+            *row,
+        )
+
+    console.print()
+    console.print(table)
+
+    out = save(results, provider=getattr(client, "label", ""), host=client.host)
+    console.print(f"[dim]Saved:[/dim] [white]{out}[/white]")
+    if (out / "arena.html").exists():
+        console.print(
+            f"[dim]Gallery:[/dim] [white]{out / 'arena.html'}[/white] "
+            "[dim]· click a preview to open the generated page[/dim]"
+        )
+        # A run whose render checks were all skipped still scores 100%, so say
+        # so plainly rather than letting the score imply the pages were judged.
+        workspaces = [
+            task.get("workspace") or {}
+            for result in results
+            for task in result.get("tasks") or []
+        ]
+        skipped = sum(
+            workspace.get("render_status") == "skipped"
+            and not workspace.get("build_status")
+            for workspace in workspaces
+        )
+        if skipped:
+            console.print(
+                f"[yellow]Skipped {skipped} render check(s):[/yellow] "
+                "[dim]this machine could not render them, so they were not "
+                "scored against the model · run 'benchkit render-check'[/dim]"
+            )
+        # mc-arena scores the block list, so a build without pictures is a
+        # missing illustration rather than an unjudged task.
+        unillustrated = sum(
+            workspace.get("render_status") == "skipped"
+            and bool(workspace.get("build_status"))
+            for workspace in workspaces
+        )
+        if unillustrated:
+            console.print(
+                f"[yellow]Rendered no image for {unillustrated} build(s):[/yellow] "
+                "[dim]the builds were still scored from their block lists · "
+                "run 'benchkit render-check'[/dim]"
+            )
+    if failure:
+        sys.exit(1)
+
+
+def _perf(args: argparse.Namespace) -> None:
+    if args.demo:
+        console.print("[red]Performance profiling is unavailable in demo mode.[/red]")
+        sys.exit(1)
+    if not args.perf_model:
+        console.print("[red]A model name is required:[/red] benchkit perf MODEL")
+        sys.exit(1)
+    if args.gen < 1 or args.reps < 1 or args.warmups < 0:
+        console.print(
+            "[red]Perf settings require --gen and --reps above zero, "
+            "and --warmups at least zero.[/red]"
+        )
+        sys.exit(1)
+    try:
+        depths = parse_depths(args.depths)
+        client = _client(args)
+    except ValueError as exc:
+        console.print(f"[red]Configuration error:[/red] {exc}")
+        sys.exit(1)
+
+    console.print(f"[bold]BenchKit perf[/bold] [dim]{client.host}[/dim]")
+    try:
+        models = client.list_models()
+    except Exception as exc:
+        console.print(f"[red]Connection failed:[/red] {exc}")
+        sys.exit(1)
+
+    model_meta = next(
+        (model for model in models if model.get("name") == args.perf_model),
+        None,
+    )
+    if model_meta is None:
+        console.print(f"[red]Unknown model:[/red] {args.perf_model}")
+        console.print(
+            "[dim]Available: "
+            + ", ".join(str(model.get("name")) for model in models)
+            + "[/dim]"
+        )
+        sys.exit(1)
+    if client.provider != "openai":
+        console.print(
+            "[red]Perf v1 requires an OpenAI-compatible llama.cpp/llama-swap "
+            "endpoint.[/red]"
+        )
+        sys.exit(1)
+
+    model_status = str(model_meta.get("status") or "").strip().lower()
+    if model_status == "unloaded":
+        console.print(
+            "[yellow]Model status: unloaded[/yellow] "
+            "[dim]· the first request may measure a cold start[/dim]"
+        )
+    elif model_status == "loaded":
+        console.print(
+            "[green]Model status: loaded[/green] "
+            "[dim]· the model is already resident, so this is a warm test[/dim]"
+        )
+    elif model_status:
+        console.print(
+            f"[dim]Model status: {model_status} · cold/warm state is not known[/dim]"
+        )
+    else:
+        console.print(
+            "[dim]Model status: unavailable · cold/warm state is not known[/dim]"
+        )
+
+    config = PerfConfig(
+        model=args.perf_model,
+        depths=depths,
+        gen_tokens=args.gen,
+        reps=args.reps,
+        warmups=args.warmups,
+        config_note=args.config_note,
+    )
+    try:
+        profile = run_profile(
+            client,
+            config,
+            model_meta=model_meta,
+            progress=lambda message: console.print(f"[dim]· {message}[/dim]"),
+        )
+    except Exception as exc:
+        console.print(f"[red]Performance profile failed:[/red] {exc}")
+        sys.exit(1)
+
+    table = Table(
+        box=box.MINIMAL,
+        border_style="dim",
+        header_style="bold",
+        pad_edge=False,
+    )
+    table.add_column("Depth")
+    table.add_column("Input", justify="right")
+    table.add_column("PP tok/s", justify="right")
+    table.add_column("TG tok/s", justify="right")
+    table.add_column("TTFT", justify="right")
+    table.add_column("Wall", justify="right")
+    table.add_column("Overhead", justify="right")
+    table.add_column("Runs", justify="right")
+    for case in profile["cases"]:
+        inaccurate = not case.get("depth_within_tolerance", True)
+        depth = case["depth_label"]
+        if inaccurate:
+            depth = f"[yellow]{depth}![/yellow]"
+        table.add_row(
+            depth,
+            f"{case['actual_input_tokens']:,}" if case["actual_input_tokens"] else "—",
+            (f"{case['pp_tps']['median']:.1f}" if case.get("pp_reliable") else "n/a"),
+            f"{case['tg_tps']['median']:.1f}",
+            f"{case['ttft_s']['median']:.3f}s",
+            f"{case['wall_time_s']['median']:.2f}s",
+            (
+                f"{case['overhead_s']['median']:.3f}s"
+                if case.get("overhead_available")
+                else "—"
+            ),
+            f"{case['successful_reps']}/{args.reps}",
+        )
+    console.print()
+    console.print(table)
+
+    sweep = profile.get("sweep_summary") or {}
+    console.print(
+        "[dim]Sweep median · "
+        f"TG {float(sweep.get('tg_tps_median') or 0):.1f} tok/s · "
+        f"PP {float(sweep.get('pp_tps_median') or 0):.1f} tok/s"
+        "[/dim]"
+    )
+
+    calibrated = [
+        case for case in profile["cases"] if case.get("depth_method") == "calibrated"
+    ]
+    inaccurate = [
+        case
+        for case in profile["cases"]
+        if not case.get("depth_within_tolerance", True)
+    ]
+    estimated = [
+        case for case in profile["cases"] if case.get("depth_method") == "estimated"
+    ]
+    if calibrated:
+        console.print(
+            "[dim]Depth method: calibrated · native tokenizer is not exposed "
+            "by the server; server-reported input tokens were used.[/dim]"
+        )
+    if inaccurate:
+        labels = ", ".join(case["depth_label"] for case in inaccurate)
+        console.print(
+            f"[bold yellow]! Depth tolerance not reached for: {labels}. "
+            "Use the Actual Input column when comparing results.[/bold yellow]"
+        )
+    if estimated:
+        labels = ", ".join(case["depth_label"] for case in estimated)
+        console.print(
+            f"[bold yellow]Depth calibration unavailable for: {labels}. "
+            "Use the Actual Input column.[/bold yellow]"
+        )
+
+    drift = profile.get("drift_check") or {}
+    if drift.get("error"):
+        console.print(f"[yellow]Drift check failed:[/yellow] {drift['error']}")
+    elif drift:
+        message = (
+            f"Drift check · {drift['depth_label']} TG "
+            f"{drift['initial_tg_tps']:.1f} → {drift['final_tg_tps']:.1f} tok/s "
+            f"({drift['change']:+.1%})"
+            + ("" if drift.get("stable") else " · possible warmup/drift anomaly")
+        )
+        console.print(
+            f"[dim]{message}[/dim]"
+            if drift.get("stable")
+            else f"[bold yellow]{message}[/bold yellow]"
+        )
+
+    output = save_profile(profile)
+    console.print(
+        f"[dim]Saved:[/dim] [white]{output}[/white] "
+        "[dim]· perf.json · csv · md · html[/dim]"
+    )
+    console.print(
+        "[dim]Tip: Run the ruler benchmark to verify how much of the configured "
+        "context window is actually usable.[/dim]"
+    )
+
+
+def main(argv: list[str] | None = None) -> None:
+    # Only the working directory's .env: walking up to parent directories would
+    # let a file in a shared parent such as /tmp redirect the host and API key.
+    load_dotenv(".env")
+    args = _parse_args(sys.argv[1:] if argv is None else argv)
+
+    if args.list:
+        _list_benchmarks()
+        return
+
+    if args.command == "render-check":
+        _render_check()
+        return
+
+    if args.command == "datasets":
+        _datasets(args)
+        return
+
+    if args.command == "perf":
+        _perf(args)
+        return
+
+    if args.command == "history":
+        if not 0 <= args.port <= 65535:
+            console.print("[red]--port must be between 0 and 65535.[/red]")
+            sys.exit(1)
+        serve_history(
+            args.results_dir or ["results"],
+            port=args.port,
+            open_browser=not args.no_open,
+        )
+        return
+
+    if args.headless:
+        _headless(args)
+        return
+
+    run_tui(demo=args.demo, host=args.host)
+
+
+if __name__ == "__main__":
+    main()