benchkit-cli 0.1.0__py3-none-any.whl

benchkit/tui/screens/connect.py

@@ -0,0 +1,191 @@
+"""Connection screen: pick a host/provider and discover models."""
+
+from __future__ import annotations
+
+import os
+
+from textual import events, work
+from textual.app import ComposeResult
+from textual.binding import Binding
+from textual.containers import Horizontal, Vertical, VerticalScroll
+from textual.screen import Screen
+from textual.widgets import Button, Footer, Header, Input, Label, Select, Static
+
+from benchkit.client import DEFAULT_HOST, InferenceClient
+from benchkit.demo import DemoClient
+from benchkit.tui.widgets import Banner
+
+PROVIDERS = [
+    ("auto detect", "auto"),
+    ("OpenAI compatible", "openai"),
+    ("Ollama", "ollama"),
+]
+
+HINT = (
+    "Point BenchKit at any OpenAI-compatible server (llama.cpp, llama-swap, vLLM, "
+    "LM Studio) or a native Ollama host."
+)
+
+
+class ConnectScreen(Screen[None]):
+    """First screen: configure the endpoint and connect."""
+
+    BINDINGS = [
+        Binding("ctrl+r", "connect", "Connect"),
+        Binding("f5", "connect", "Connect", show=False),
+        Binding("f3", "demo", "Demo mode"),
+        Binding("escape", "app.quit", "Quit", show=False),
+    ]
+
+    def compose(self) -> ComposeResult:
+        yield Header(show_clock=True)
+        with VerticalScroll(id="connect-root"):
+            yield Banner(id="banner")
+            yield Static(
+                "Benchmark your local LLMs — not vibes, actual scores.", id="tagline"
+            )
+            with Vertical(id="connect-card"):
+                yield Static("Connection", classes="card-title")
+                with Vertical(id="connect-form"):
+                    with Horizontal(classes="field"):
+                        yield Label("Host", classes="field-label")
+                        yield Input(placeholder=DEFAULT_HOST, id="host", compact=True)
+                    with Horizontal(classes="field"):
+                        yield Label("Provider", classes="field-label")
+                        yield Select(
+                            PROVIDERS,
+                            value="auto",
+                            allow_blank=False,
+                            id="provider",
+                            compact=True,
+                        )
+                    with Horizontal(classes="field"):
+                        yield Label("API key", classes="field-label")
+                        yield Input(
+                            placeholder="optional",
+                            password=True,
+                            id="api-key",
+                            compact=True,
+                        )
+                    with Horizontal(classes="field"):
+                        yield Label("Task timeout", classes="field-label")
+                        yield Input(placeholder="300", id="timeout", compact=True)
+                with Horizontal(id="connect-actions"):
+                    yield Button("Connect", id="connect", variant="primary")
+                    yield Button("Quit", id="quit", variant="error")
+                yield Static(HINT, id="connect-status")
+        yield Footer()
+
+    def on_resize(self, event: events.Resize) -> None:
+        # The wordmark is 62 cells wide, so it goes first on a small terminal.
+        size = event.size
+        self.set_class(size.height < 30 or size.width < 66, "compact")
+
+    def on_mount(self) -> None:
+        defaults = _env_defaults()
+        host = self.app.host_override or defaults["host"]
+        self.query_one("#host", Input).value = host
+        self.query_one("#provider", Select).value = defaults["provider"]
+        self.query_one("#api-key", Input).value = defaults["api_key"]
+        self.query_one("#timeout", Input).value = defaults["timeout"]
+
+        if self.app.demo:
+            self.set_timer(0.1, self.action_demo)
+        else:
+            self.set_timer(0.1, self.action_connect)
+
+    # Actions ----------------------------------------------------------
+
+    def action_connect(self) -> None:
+        host = self.query_one("#host", Input).value.strip() or DEFAULT_HOST
+        provider = self.query_one("#provider", Select).value
+        api_key = self.query_one("#api-key", Input).value.strip() or None
+        raw_timeout = self.query_one("#timeout", Input).value.strip() or "300"
+        try:
+            timeout = float(raw_timeout)
+        except ValueError:
+            self._status(f"Timeout must be a number, got {raw_timeout!r}", "error")
+            return
+
+        self._busy(True)
+        self._status(f"Connecting to {host} …", "working")
+        self._connect(host.rstrip("/"), str(provider), api_key, timeout)
+
+    def action_demo(self) -> None:
+        client = DemoClient()
+        self._status("Demo mode — offline models, no server required.", "ok")
+        self.app.show_setup(client, client.list_models())
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if event.button.id == "connect":
+            self.action_connect()
+        elif event.button.id == "quit":
+            self.app.exit()
+
+    def on_input_submitted(self) -> None:
+        self.action_connect()
+
+    # Worker -----------------------------------------------------------
+
+    @work(thread=True, exclusive=True, group="connect")
+    def _connect(
+        self, host: str, provider: str, api_key: str | None, timeout: float
+    ) -> None:
+        try:
+            client = InferenceClient(host, provider, api_key, timeout)
+            models = client.list_models()
+        except Exception as exc:
+            self.app.call_from_thread(self._failed, f"{type(exc).__name__}: {exc}")
+            return
+        self.app.call_from_thread(self._connected, client, models)
+
+    def _connected(self, client: InferenceClient, models: list[dict]) -> None:
+        self._busy(False)
+        if not models:
+            self._status(
+                f"Connected to {client.host} but the server reports no models.", "error"
+            )
+            return
+        self._status(f"{len(models)} model(s) via {client.label}", "ok")
+        self.app.show_setup(client, models)
+
+    def _failed(self, message: str) -> None:
+        self._busy(False)
+        self._status(
+            f"{message}\nCheck the host and that the server is running, "
+            "or press F3 for demo mode.",
+            "error",
+        )
+
+    # Helpers ----------------------------------------------------------
+
+    def _busy(self, busy: bool) -> None:
+        self.query_one("#connect", Button).disabled = busy
+        self.query_one("#connect-card").set_class(busy, "busy")
+
+    def _status(self, message: str, kind: str = "") -> None:
+        status = self.query_one("#connect-status", Static)
+        status.set_classes(f"status {kind}" if kind else "status")
+        status.update(message)
+        status.scroll_visible()
+
+
+def _env_defaults() -> dict[str, str]:
+    provider = os.environ.get("BENCHKIT_PROVIDER", "auto").lower()
+    if provider not in {"auto", "openai", "ollama"}:
+        provider = "auto"
+    host = (
+        os.environ.get("BENCHKIT_HOST")
+        or os.environ.get("OPENAI_BASE_URL")
+        or os.environ.get("OLLAMA_HOST")
+        or DEFAULT_HOST
+    )
+    api_key = (
+        os.environ.get("BENCHKIT_API_KEY") or os.environ.get("OPENAI_API_KEY") or ""
+    )
+    return {
+        "host": host.rstrip("/"),
+        "provider": provider,
+        "api_key": api_key,
+        "timeout": os.environ.get("BENCHKIT_TIMEOUT", "300"),
+    }

benchkit/tui/screens/detail.py

@@ -0,0 +1,288 @@
+"""Per-run drill-down: every task of a single model x benchmark."""
+
+from __future__ import annotations
+
+from rich.text import Text
+from textual import events
+from textual.app import ComposeResult
+from textual.binding import Binding
+from textual.containers import Horizontal, Vertical
+from textual.screen import Screen
+from textual.widgets import DataTable, Footer, Header, Input
+
+from benchkit.engine import slice_label
+from benchkit.metrics import (
+    aggregate_tok_s,
+    decision_speed,
+    effective_concurrency,
+    is_decision,
+    latency_text,
+    stream_tok_s,
+)
+from benchkit.tui.formatting import (
+    fmt_count,
+    fmt_duration,
+    result_color,
+    score_color,
+    throughput_stat,
+)
+from benchkit.tui.screens.modals import TaskDetailScreen
+from benchkit.tui.widgets import SectionTitle, StatCard, apply_compact
+
+FILTERS = ["all", "failed", "passed"]
+
+
+class JobDetailScreen(Screen[None]):
+    """Task-level view of one benchmark run."""
+
+    BINDINGS = [
+        Binding("f", "cycle_filter", "Filter"),
+        Binding("slash", "focus_search", "Search"),
+        Binding("escape", "close", "Back"),
+    ]
+
+    def __init__(self, result: dict) -> None:
+        super().__init__()
+        self.result = result
+        self.filter_index = 0
+
+    def compose(self) -> ComposeResult:
+        yield Header(show_clock=True)
+        with Vertical(id="detail-root"):
+            with Horizontal(id="stat-row"):
+                yield StatCard("Score", "--", id="stat-score")
+                yield StatCard("Passed", "--", id="stat-passed")
+                yield StatCard("Loop rate", "--", id="stat-loops")
+                yield StatCard("Throughput", "--", id="stat-speed")
+                yield StatCard("Avg latency", "--", id="stat-latency")
+                yield StatCard("Duration", "--", id="stat-duration")
+            with Vertical(classes="pane", id="detail-pane"):
+                yield SectionTitle("Tasks", "enter to open", id="detail-title")
+                yield Input(placeholder="Search task id or response…", id="task-search")
+                yield DataTable(id="task-table", cursor_type="row", zebra_stripes=True)
+        yield Footer()
+
+    def on_mount(self) -> None:
+        result = self.result
+        benchmark_label = result.get("benchmark_label", result["benchmark"])
+        harness = result.get("harness_label", "Direct")
+        self.title = f"{benchmark_label} · {result['model']} · {harness}"
+        self.sub_title = f"{result['passed']}/{result['total']} passed"
+
+        table = self.query_one("#task-table", DataTable)
+        table.add_column("#", key="index", width=5)
+        table.add_column("Task", key="task")
+        table.add_column("Result", key="result", width=8)
+        table.add_column("Loop", key="loop", width=9)
+        table.add_column("Think", key="thinking", width=8)
+        table.add_column("Latency", key="latency", width=9)
+        table.add_column("Stream tok/s", key="tok_s", width=12)
+
+        score = result["score"]
+        score_hint = slice_label(result.get("slice"))
+        if result.get("score_delta_pp") is not None:
+            score_hint = (
+                f"{score_hint} · Δ {result['score_delta_pp']:+.1f} pp · "
+                f"{result.get('regressions', 0)} regression(s) · "
+                f"seed {result.get('perturbation_seed', 42)}"
+            )
+        if result.get("harness_score_delta_pp") is not None:
+            score_hint += (
+                f" · harness score Δ {result['harness_score_delta_pp']:+.1f} pp"
+                f" · loop-kill Δ {result.get('loop_kill_delta_pp', 0):+.1f} pp"
+                + (
+                    f" (initial {result.get('harness_first_score_delta_pp', 0):+.1f})"
+                    if result.get("repair_attempts")
+                    else ""
+                )
+                + " · "
+                f"{result.get('harness_gains', 0)} gain(s) · "
+                f"{result.get('harness_regressions', 0)} regression(s)"
+            )
+        if result.get("repair_attempts"):
+            score_hint += (
+                f" · repair Δ {result.get('repair_delta_pp', 0):+.1f} pp · "
+                f"{result.get('repair_successes', 0)}/"
+                f"{result.get('repair_attempted', 0)} fixed"
+            )
+        self.query_one("#stat-score", StatCard).set_state(f"{score:.1f}%", score_hint)
+        self.query_one("#stat-passed", StatCard).set_state(
+            f"{result['passed']}/{result['total']}",
+            f"{result.get('errors', 0)} error(s)",
+        )
+        self.query_one("#stat-loops", StatCard).set_state(
+            f"{result.get('loop_rate', 0):.1f}%",
+            f"{result.get('loops', 0)} loop · "
+            f"{result.get('suspected_loops', 0)} suspect · "
+            f"{result.get('recovered_loops', 0)} recovered · "
+            f"{result.get('loop_kills', 0)} killed",
+        )
+        speed = self.query_one("#stat-speed", StatCard)
+        speed.set_state(
+            *(
+                decision_speed(result)
+                if is_decision(result)
+                else throughput_stat(
+                    concurrency=result.get("concurrency", 1),
+                    aggregate=aggregate_tok_s(result),
+                    stream=stream_tok_s(result),
+                    effective=effective_concurrency(result),
+                )
+            )
+        )
+        self.query_one("#stat-latency", StatCard).set_state(latency_text(result))
+        self.query_one("#stat-duration", StatCard).set_state(
+            fmt_duration(result["total_time"])
+        )
+        self._restyle_score()
+
+        self._fill()
+        self.watch(self.app, "theme", self._theme_changed, init=False)
+        table.focus()
+
+    def _restyle_score(self) -> None:
+        color = score_color(self.result["score"], self.app.current_theme.dark)
+        self.query_one("#stat-score", StatCard).styles.color = color
+
+    def _theme_changed(self, _theme: str) -> None:
+        self._restyle_score()
+        self._fill()
+
+    def on_resize(self, event: events.Resize) -> None:
+        apply_compact(self, event.size.height)
+
+    # Rendering --------------------------------------------------------
+
+    def _visible_tasks(self) -> list[tuple[int, dict]]:
+        mode = FILTERS[self.filter_index]
+        needle = self.query_one("#task-search", Input).value.strip().lower()
+        rows = []
+        for index, task in enumerate(self.result["tasks"]):
+            if mode == "failed" and task["passed"]:
+                continue
+            if mode == "passed" and not task["passed"]:
+                continue
+            if (
+                needle
+                and needle not in task["task_id"].lower()
+                and needle not in (task.get("response", "").lower())
+                and needle not in (task.get("thinking", "").lower())
+            ):
+                continue
+            rows.append((index, task))
+        return rows
+
+    def _fill(self) -> None:
+        table = self.query_one("#task-table", DataTable)
+        table.clear()
+        rows = self._visible_tasks()
+        for index, task in rows:
+            entry = task.get("entry_point")
+            label = f"{task['task_id']} ({entry})" if entry else task["task_id"]
+            table.add_row(
+                str(index + 1),
+                label,
+                _result_cell(task, self.app.current_theme.dark),
+                _loop_cell(task),
+                (
+                    fmt_duration(task.get("thinking_time_s", 0))
+                    if task.get("thinking")
+                    else "—"
+                ),
+                fmt_duration(task.get("response_time_s", 0)),
+                f"{task.get('tok_s', 0):.1f}",
+                key=str(index),
+            )
+        self.query_one("#detail-title", SectionTitle).set_detail(
+            f"{FILTERS[self.filter_index]} · {fmt_count(len(rows))} shown · enter to open"
+        )
+
+    # Events -----------------------------------------------------------
+
+    def on_input_changed(self) -> None:
+        self._fill()
+
+    def on_input_submitted(self) -> None:
+        self.query_one("#task-table", DataTable).focus()
+
+    def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
+        index = int(str(event.row_key.value))
+        task = self.result["tasks"][index]
+        benchmark_label = self.result.get("benchmark_label", self.result["benchmark"])
+        title = (
+            f"{benchmark_label} · {self.result['model']} · "
+            f"{self.result.get('harness_label', 'Direct')}"
+        )
+        self.app.push_screen(TaskDetailScreen(title, task))
+
+    # Actions ----------------------------------------------------------
+
+    def action_cycle_filter(self) -> None:
+        self.filter_index = (self.filter_index + 1) % len(FILTERS)
+        self._fill()
+
+    def action_focus_search(self) -> None:
+        self.query_one("#task-search", Input).focus()
+
+    def action_close(self) -> None:
+        self.app.pop_screen()
+
+
+def _result_cell(task: dict, dark: bool = True) -> Text:
+    error = bool(task.get("error"))
+    passed = bool(task.get("passed"))
+    label = (
+        "LOOPKILL"
+        if task.get("loop_killed")
+        else "TIMEOUT"
+        if task.get("timed_out")
+        else "LENGTH"
+        if task.get("length_exceeded")
+        else "ERROR"
+        if error
+        else "FIXED"
+        if task.get("repaired")
+        else "PASS"
+        if passed
+        else f"{float(task.get('score', 0)):.0f}%"
+        if float(task.get("score", 0)) > 0
+        else "FAIL"
+    )
+    color = (
+        score_color(float(task.get("score", 0)), dark)
+        if not passed and float(task.get("score", 0)) > 0 and not error
+        else result_color(passed, error, dark)
+    )
+    return Text(label, style=f"bold {color}")
+
+
+def _loop_cell(task: dict) -> Text:
+    state = task.get("loop_state", "unavailable")
+    source = task.get("loop_source", "none")
+    label = (
+        "RECOVERED"
+        if task.get("recovered_cycle")
+        else "OUTPUT"
+        if state == "looping" and source == "answer"
+        else "LOOPING"
+        if state == "looping"
+        else "SUSPECT"
+        if state == "suspected"
+        else "NO TRACE"
+        if state == "clear" and source == "answer"
+        else "CLEAR"
+        if state == "clear"
+        else "NO TRACE"
+    )
+    style = (
+        "bold yellow"
+        if task.get("recovered_cycle")
+        else "bold red"
+        if state == "looping"
+        else "bold yellow"
+        if state == "suspected"
+        else "green"
+        if state == "clear" and source != "answer"
+        else "dim"
+    )
+    return Text(label, style=style)

benchkit/tui/screens/help.py

@@ -0,0 +1,84 @@
+"""Keyboard reference."""
+
+from __future__ import annotations
+
+from textual.app import ComposeResult
+from textual.binding import Binding
+from textual.containers import VerticalScroll
+from textual.screen import ModalScreen
+from textual.widgets import Footer, Markdown, Static
+
+HELP = """
+### Everywhere
+
+| Key | Action |
+| --- | --- |
+| `?` / `F1` | This help |
+| `Ctrl+P` | Command palette |
+| `F2` | Dogi light / dark theme |
+| `Ctrl+Q` | Quit |
+| `Tab` / `Shift+Tab` | Move between panes |
+
+### Connect
+
+| Key | Action |
+| --- | --- |
+| `Ctrl+R` | Connect / retry |
+| `F3` | Demo mode (no server needed) |
+| `Enter` | Connect from any field |
+
+### Setup
+
+| Key | Action |
+| --- | --- |
+| `Space` / `Enter` | Toggle the highlighted row |
+| `a` / `n` / `i` | Select all / clear / invert the focused list |
+| `l` | Task limit for the highlighted benchmark |
+| `p` | Toggle the paired choice-order run |
+| `v` | Check chat templates (selected models, or all) |
+| `d` | Datasets: see what is installed and install what is missing |
+| `/` | Jump to the filter box |
+| `s` / `F5` | Start the run |
+| `Esc` | Back to the connection screen |
+
+Limits accept `20` (first 20), `-20` (last 20) and `40-80` (a range).
+Choice-order runs each selected task clean and with deterministically shuffled options.
+
+### Run
+
+| Key | Action |
+| --- | --- |
+| `p` | Pause / resume between tasks |
+| `k` | Skip the current model × benchmark |
+| `x` | Stop the run and keep what finished |
+| `f` | Show failures only |
+| `t` | Follow the newest task (auto-scroll) |
+| `Enter` | Inspect the highlighted task |
+
+### Results
+
+| Key | Action |
+| --- | --- |
+| `Enter` | Drill into a run's tasks |
+| `s` / `m` / `t` | Sort by score / model / duration |
+| `c` | Copy the results folder path |
+| `r` | Configure another run |
+| `Esc` | Back to setup |
+"""
+
+
+class HelpScreen(ModalScreen[None]):
+    BINDINGS = [
+        Binding("escape", "close", "Close"),
+        Binding("question_mark", "close", "Close", show=False),
+        Binding("q", "close", "Close", show=False),
+    ]
+
+    def compose(self) -> ComposeResult:
+        with VerticalScroll(id="help-dialog", classes="dialog"):
+            yield Static("Keyboard reference", classes="card-title")
+            yield Markdown(HELP)
+        yield Footer()
+
+    def action_close(self) -> None:
+        self.dismiss(None)