benchkit-cli 0.1.0__py3-none-any.whl

benchkit/tui/__init__.py

@@ -0,0 +1,5 @@
+"""BenchKit terminal user interface."""
+
+from benchkit.tui.app import BenchKitApp, run_tui
+
+__all__ = ["BenchKitApp", "run_tui"]

benchkit/tui/app.py

@@ -0,0 +1,87 @@
+"""The BenchKit terminal application."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from textual.app import App
+from textual.binding import Binding
+
+from benchkit.engine import JobSpec
+from benchkit.tui.screens.connect import ConnectScreen
+from benchkit.tui.screens.help import HelpScreen
+from benchkit.tui.screens.results import ResultsScreen
+from benchkit.tui.screens.run import RunScreen
+from benchkit.tui.screens.setup import SetupScreen
+from benchkit.tui.theme import (
+    ALTERNATE_THEME,
+    DEFAULT_THEME,
+    THEMES,
+    VARIABLE_DEFAULTS,
+)
+
+
+class BenchKitApp(App[None]):
+    """Full-screen benchmark cockpit: connect, configure, run, inspect."""
+
+    CSS_PATH = "app.tcss"
+    TITLE = "BenchKit"
+    SUB_TITLE = "benchmark your local LLMs"
+    ENABLE_COMMAND_PALETTE = True
+
+    BINDINGS = [
+        Binding("question_mark", "help", "Help"),
+        Binding("f1", "help", "Help", show=False),
+        Binding("ctrl+q", "quit", "Quit", priority=True, show=True),
+        Binding("f2", "toggle_theme", "Theme", show=False),
+    ]
+
+    def __init__(self, *, demo: bool = False, host: str | None = None) -> None:
+        super().__init__()
+        for theme in THEMES:
+            self.register_theme(theme)
+        self.theme = DEFAULT_THEME
+        self.demo = demo
+        self.host_override = host
+        self.client = None
+        self.models: list[dict] = []
+        self.last_output: Path | None = None
+
+    def get_theme_variable_defaults(self) -> dict[str, str]:
+        return dict(VARIABLE_DEFAULTS)
+
+    def on_mount(self) -> None:
+        self.push_screen(ConnectScreen())
+
+    # Navigation -------------------------------------------------------
+
+    def show_setup(self, client, models: list[dict]) -> None:
+        self.client = client
+        self.models = models
+        self.push_screen(SetupScreen())
+
+    def start_run(self, jobs: list[JobSpec], *, force_unload: bool = False) -> None:
+        self.push_screen(RunScreen(jobs, force_unload=force_unload))
+
+    def show_results(self, results: list[dict], output: Path | None) -> None:
+        self.last_output = output
+        self.switch_screen(ResultsScreen(results, output))
+
+    def back_to_setup(self) -> None:
+        """Return to the configuration screen from anywhere downstream."""
+        while len(self.screen_stack) > 2 and not isinstance(self.screen, SetupScreen):
+            self.pop_screen()
+
+    # Actions ----------------------------------------------------------
+
+    def action_help(self) -> None:
+        if not isinstance(self.screen, HelpScreen):
+            self.push_screen(HelpScreen())
+
+    def action_toggle_theme(self) -> None:
+        self.theme = ALTERNATE_THEME if self.theme == DEFAULT_THEME else DEFAULT_THEME
+        self.notify(f"Theme: {self.theme}", timeout=2)
+
+
+def run_tui(*, demo: bool = False, host: str | None = None) -> None:
+    BenchKitApp(demo=demo, host=host).run()

benchkit/tui/app.tcss

@@ -0,0 +1,512 @@
+/* BenchKit TUI stylesheet */
+
+Screen {
+    background: $background;
+}
+
+Header {
+    background: $bk-raise;
+}
+
+Footer {
+    background: $bk-raise;
+}
+
+.pane {
+    background: $surface;
+    border: round $bk-border;
+    padding: 0 1;
+    height: 1fr;
+}
+
+.pane:focus-within {
+    border: round $primary;
+}
+
+.card-title {
+    text-style: bold;
+    padding: 0 0 1 0;
+}
+
+.hint {
+    color: $text-muted;
+}
+
+.field {
+    height: 1;
+    margin: 0 0 1 0;
+}
+
+.field-label {
+    width: 12;
+    padding: 0 1 0 0;
+    color: $text-muted;
+}
+
+.field-label.wide {
+    width: 22;
+}
+
+.filter {
+    border: none;
+    background: $bk-raise;
+    height: 1;
+    margin: 0 0 1 0;
+    padding: 0 1;
+}
+
+.filter:focus {
+    background: $primary 15%;
+}
+
+SelectionList {
+    background: $surface;
+    height: 1fr;
+    padding: 0;
+}
+
+DataTable {
+    height: 1fr;
+    background: $surface;
+}
+
+DataTable > .datatable--header {
+    background: $bk-raise;
+    color: $text-muted;
+    text-style: bold;
+}
+
+DataTable > .datatable--cursor {
+    background: $bk-selection;
+    color: $foreground;
+    text-style: bold;
+}
+
+/* ---------------------------------------------------------------- Connect */
+
+#connect-root {
+    align: center middle;
+    height: 1fr;
+    padding: 1 2;
+}
+
+/* Every child is pinned to the card width so `align: center middle` has a
+   group narrower than the screen to centre. A full-width child (Static
+   defaults to one) would silently pin the group to the left edge. */
+#banner {
+    width: 72;
+    max-width: 100%;
+    color: $primary;
+    text-style: bold;
+    margin: 0 0 1 0;
+}
+
+#tagline {
+    width: 72;
+    max-width: 100%;
+    color: $text-muted;
+    margin: 0 0 1 0;
+}
+
+#connect-card {
+    width: 72;
+    max-width: 100%;
+    height: auto;
+    background: $surface;
+    border: round $bk-border;
+    padding: 1 2;
+}
+
+#connect-card.busy {
+    border: round $primary;
+}
+
+#connect-form {
+    height: auto;
+}
+
+#connect-form Input,
+#connect-form Select {
+    width: 1fr;
+    background: $bk-raise;
+}
+
+#connect-actions {
+    height: auto;
+    padding: 1 0 0 0;
+}
+
+#connect-actions Button {
+    margin: 0 2 0 0;
+}
+
+#connect-status {
+    height: auto;
+    padding: 1 0 0 0;
+    color: $text-muted;
+}
+
+#connect-status.working {
+    color: $accent;
+}
+
+#connect-status.ok {
+    color: $success;
+}
+
+#connect-status.error {
+    color: $error;
+}
+
+/* ------------------------------------------------------------------ Setup */
+
+#setup-body {
+    height: 1fr;
+    padding: 1 1 0 1;
+}
+
+#model-pane {
+    width: 5fr;
+    margin: 0 1 0 0;
+}
+
+#bench-pane {
+    width: 7fr;
+}
+
+/* Sized to the rows it holds, so a filtered list does not leave a tall gap. */
+#bench-list {
+    height: auto;
+    max-height: 1fr;
+}
+
+#setup-options {
+    height: 1;
+    padding: 0 2;
+    margin: 1 0 0 0;
+}
+
+#setup-options .field-label,
+#setup-harness .field-label,
+#setup-perturbation .field-label,
+#setup-unload .field-label {
+    padding: 0 1 0 0;
+}
+
+.limit-input {
+    width: 14;
+    border: none;
+    background: $bk-raise;
+    padding: 0 1;
+    height: 1;
+    margin: 0 2 0 0;
+}
+
+#setup-options .hint {
+    text-wrap: nowrap;
+    text-overflow: ellipsis;
+}
+
+#setup-harness {
+    height: 3;
+    padding: 0 2;
+}
+
+#setup-harness Select {
+    width: 22;
+    height: 3;
+    margin: -1 2 0 0;
+}
+
+.repair-input {
+    width: 8;
+    height: 3;
+    margin: -1 2 0 0;
+}
+
+#setup-harness .hint {
+    padding: 0 0 0 0;
+    text-wrap: nowrap;
+    text-overflow: ellipsis;
+}
+
+#setup-perturbation {
+    height: 1;
+    padding: 0 2;
+}
+
+#choice-order {
+    width: 29;
+    height: 1;
+}
+
+#perturbation-seed-label {
+    width: 6;
+    padding: 0 1 0 0;
+}
+
+.seed-input {
+    width: 9;
+    height: 1;
+    border: none;
+    background: $bk-raise;
+    padding: 0 1;
+    margin: 0 2 0 0;
+}
+
+#setup-perturbation .hint {
+    text-wrap: nowrap;
+    text-overflow: ellipsis;
+}
+
+#setup-unload {
+    height: 1;
+    padding: 0 2;
+}
+
+#force-unload {
+    width: 32;
+    height: 1;
+}
+
+#setup-unload .hint {
+    text-wrap: nowrap;
+    text-overflow: ellipsis;
+}
+
+#setup-footer {
+    height: 3;
+    padding: 0 2;
+    align-vertical: middle;
+    background: $bk-raise;
+}
+
+#plan-summary {
+    width: 1fr;
+    padding: 1 0 0 0;
+}
+
+#template-check-dialog {
+    width: 92%;
+    height: 70%;
+}
+
+#template-check-table {
+    height: 1fr;
+    margin: 1 0;
+}
+
+#datasets-dialog {
+    width: 92%;
+    height: 80%;
+}
+
+#datasets-table {
+    height: 1fr;
+    margin: 1 0 0 0;
+}
+
+/* -------------------------------------------------------------------- Run */
+
+#run-root {
+    height: 1fr;
+    padding: 1 1 0 1;
+}
+
+#run-head {
+    height: 1;
+    padding: 0 1;
+}
+
+#run-title {
+    width: 1fr;
+}
+
+.state {
+    width: auto;
+    padding: 0 1;
+    text-style: bold;
+}
+
+.state.running {
+    background: $success 20%;
+    color: $success;
+}
+
+.state.paused {
+    background: $warning 20%;
+    color: $warning;
+}
+
+.state.evaluating {
+    background: $warning 20%;
+    color: $warning;
+}
+
+.state.looping {
+    background: $error 20%;
+    color: $error;
+}
+
+.state.done {
+    background: $accent 20%;
+    color: $accent;
+}
+
+#progress-block {
+    height: auto;
+    padding: 1 1 0 1;
+}
+
+#progress-block Static {
+    color: $text-muted;
+}
+
+#progress-block ProgressBar {
+    width: 100%;
+    margin: 0 0 1 0;
+}
+
+#progress-block Bar {
+    width: 1fr;
+}
+
+ProgressBar > Bar > .bar--bar {
+    color: $primary;
+}
+
+ProgressBar > Bar > .bar--complete {
+    color: $success;
+}
+
+#stat-row {
+    height: 5;
+    padding: 0 1;
+}
+
+#run-body {
+    height: 1fr;
+    padding: 1 0 0 0;
+}
+
+#queue-pane {
+    width: 1fr;
+    margin: 0 1 0 0;
+}
+
+#task-pane {
+    width: 1fr;
+}
+
+/* --------------------------------------------------------------- Results */
+
+#results-root, #detail-root {
+    height: 1fr;
+    padding: 1 1 0 1;
+}
+
+#results-pane, #detail-pane {
+    height: 1fr;
+}
+
+#output-path {
+    height: 1;
+    padding: 0 2;
+}
+
+#task-search {
+    border: none;
+    background: $bk-raise;
+    height: 1;
+    margin: 0 0 1 0;
+    padding: 0 1;
+}
+
+/* --------------------------------------------------------------- Modals */
+
+LimitScreen, TaskDetailScreen, ConfirmScreen, HelpScreen {
+    align: center middle;
+    background: $background 70%;
+}
+
+.dialog {
+    background: $surface;
+    border: round $primary;
+    padding: 1 2;
+}
+
+#limit-dialog {
+    width: 64;
+    height: auto;
+}
+
+#limit-dialog Input {
+    margin: 1 0;
+}
+
+#confirm-dialog {
+    width: 56;
+    height: auto;
+}
+
+#task-dialog {
+    width: 90%;
+    height: 85%;
+}
+
+#task-dialog TabbedContent {
+    height: 1fr;
+}
+
+#task-dialog TextArea {
+    height: 1fr;
+    border: none;
+}
+
+#help-dialog {
+    width: 84;
+    height: 80%;
+}
+
+.dialog-actions {
+    height: auto;
+    padding: 1 0 0 0;
+}
+
+.dialog-actions Button {
+    margin: 0 2 0 0;
+}
+
+/* ------------------------------------------------ Short-terminal fallback */
+
+.compact #banner {
+    display: none;
+}
+
+.compact #tagline {
+    display: none;
+}
+
+.compact #stat-row {
+    height: 3;
+}
+
+.compact StatCard {
+    height: 2;
+}
+
+.compact .stat-hint {
+    display: none;
+}
+
+.compact #progress-block {
+    padding: 0 1;
+}
+
+.compact #progress-block ProgressBar {
+    margin: 0;
+}
+
+.compact #run-body {
+    padding: 0;
+}

benchkit/tui/formatting.py

@@ -0,0 +1,84 @@
+"""Small formatting helpers shared across TUI screens."""
+
+from __future__ import annotations
+
+from benchkit.tui.theme import score_palette
+
+
+def fmt_duration(seconds: float | None) -> str:
+    """Compact duration: ``1h 04m``, ``4m 12s``, ``42s`` or ``0.8s``."""
+    if seconds is None:
+        return "--"
+    if seconds < 0:
+        return "--"
+    if seconds < 10:
+        return f"{seconds:.1f}s"
+
+    total = round(seconds)
+    if total < 60:
+        return f"{total}s"
+    if total < 3600:
+        return f"{total // 60}m {total % 60:02d}s"
+    return f"{total // 3600}h {(total % 3600) // 60:02d}m"
+
+
+def fmt_size(size: object) -> str:
+    """Render a model size in GB when the server reports one."""
+    if isinstance(size, (int, float)) and size > 0:
+        return f"{size / (1024**3):.1f} GB"
+    return ""
+
+
+def fmt_count(value: int) -> str:
+    return f"{value:,}"
+
+
+def throughput_stat(
+    *,
+    concurrency: int,
+    aggregate: float,
+    stream: float,
+    effective: float,
+    precision: int = 1,
+) -> tuple[str, str]:
+    """Format throughput, reserving aggregate metrics for parallel runs."""
+    if concurrency > 1:
+        return (
+            f"{aggregate:.{precision}f} tok/s",
+            f"aggregate · {stream:.{precision}f} stream · {effective:.2f}x effective",
+        )
+    return f"{stream:.{precision}f} tok/s", "single stream"
+
+
+def score_class(score: float) -> str:
+    """CSS class name matching a score band."""
+    if score >= 80:
+        return "score-high"
+    if score >= 50:
+        return "score-mid"
+    return "score-low"
+
+
+def score_color(score: float, dark: bool = True) -> str:
+    """Score colour from the active Dogi palette."""
+    palette = score_palette(dark)
+    if score >= 80:
+        return palette["high"]
+    if score >= 50:
+        return palette["mid"]
+    return palette["low"]
+
+
+def result_color(passed: bool, error: bool = False, dark: bool = True) -> str:
+    """Colour for a single task outcome."""
+    palette = score_palette(dark)
+    if error:
+        return palette["mid"]
+    return palette["high"] if passed else palette["low"]
+
+
+def bar(fraction: float, width: int = 12) -> str:
+    """A tiny inline bar for table cells."""
+    fraction = max(0.0, min(1.0, fraction))
+    filled = round(fraction * width)
+    return "█" * filled + "░" * (width - filled)

benchkit/tui/screens/__init__.py

@@ -0,0 +1 @@
+"""Screens that make up the BenchKit TUI."""