slmeval-cli 0.1.0__py3-none-any.whl

slmeval/report/progress.py

@@ -0,0 +1,257 @@
+"""The live progress view, as an :class:`~slmeval.core.runner.Observer`.
+
+A matrix run takes twenty minutes. The user has to be able to see it working
+and form an opinion before it ends - a silent terminal for twenty minutes is
+indistinguishable from a hang, and the first thing anyone does about a hang is
+``Ctrl-C``.
+
+It lives here rather than in the runner because the runner must not print. That
+keeps the orchestration testable without a terminal, and it means this whole
+file can be swapped for a JSON-lines observer in CI without touching the
+scheduler.
+
+Three details that matter more than they look:
+
+- **One bar per candidate.** The interesting question mid-run is not "how far
+  along" but "is this candidate collapsing", and a single aggregate bar cannot
+  answer it.
+- **Live pass / fail / error counters, with errors separate.** A harness error
+  must be visibly not a model failure even while the run is still going.
+- **Non-TTY degrades to plain lines.** Piped into a CI log, a repainting
+  progress bar becomes thousands of escape sequences; the same information goes
+  out one line at a time instead.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from types import TracebackType
+
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TaskID,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+from rich.text import Text
+
+from slmeval.cli.render import console as make_console
+from slmeval.cli.render import symbols
+from slmeval.core.runner import RunPlan, RunSummary, UnitResult, WorkUnit
+from slmeval.storage.models import Outcome
+
+_QUIET_OUTCOMES = frozenset({Outcome.PASSED, Outcome.UNCLASSIFIED})
+
+
+@dataclass
+class Tally:
+    """Live counters. ``errors`` is ours, and stays apart from ``failed``."""
+
+    passed: int = 0
+    failed: int = 0
+    errors: int = 0
+    uncovered: int = 0
+    unsafe: int = 0
+
+    def add(self, result: UnitResult) -> None:
+        if result.outcome is Outcome.PASSED:
+            self.passed += 1
+        elif result.outcome is Outcome.HARNESS_ERROR:
+            self.errors += 1
+        elif result.outcome is Outcome.UNCOVERED:
+            self.uncovered += 1
+        else:
+            self.failed += 1
+        if result.outcome is Outcome.FORBIDDEN_TOOL:
+            self.unsafe += 1
+
+    def render(self) -> str:
+        parts = [f"{self.passed} pass", f"{self.failed} fail"]
+        if self.errors:
+            parts.append(f"{self.errors} harness")
+        if self.uncovered:
+            parts.append(f"{self.uncovered} uncovered")
+        if self.unsafe:
+            parts.append(f"{self.unsafe} UNSAFE")
+        return " · ".join(parts)
+
+
+@dataclass
+class ProgressObserver:
+    """Renders a run as it happens. Use as a context manager."""
+
+    console: Console = field(default_factory=make_console)
+    ascii_mode: bool | None = None
+    quiet: bool = False
+
+    _progress: Progress | None = field(default=None, init=False, repr=False)
+    _tasks: dict[str, TaskID] = field(default_factory=dict, init=False, repr=False)
+    _overall: TaskID | None = field(default=None, init=False, repr=False)
+    _tally: Tally = field(default_factory=Tally, init=False, repr=False)
+    _plain: bool = field(default=False, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        # Rich's own terminal detection, not an environment guess: it already
+        # accounts for NO_COLOR, dumb terminals and redirected output.
+        self._plain = self.quiet or not self.console.is_terminal
+
+    # ------------------------------------------------------------- lifecycle
+
+    def __enter__(self) -> ProgressObserver:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self._progress is not None:
+            self._progress.stop()
+            self._progress = None
+
+    # -------------------------------------------------------------- observer
+
+    def on_plan(self, plan: RunPlan) -> None:
+        self.console.print()
+        self.console.print(
+            Text(f"  {plan.name}", style="bold"),
+            Text(
+                f" · {len(plan.matrix.candidate_ids)} candidates"
+                f" · {len(plan.matrix.case_ids)} cases"
+                f" x {plan.matrix.repeats} runs"
+                f" · {plan.matrix.planned} executions",
+                style="dim",
+            ),
+        )
+        detail = f"  {plan.run_id} · {plan.tool_mode} · {plan.scheduling}"
+        if plan.limits:
+            detail += f" · limits {plan.limits}"
+        if plan.matrix.resumed:
+            detail += f" · resuming, {len(plan.matrix.resumed)} already done"
+        self.console.print(Text(detail, style="dim"))
+        self.console.print()
+
+        if self._plain:
+            return
+        self._progress = Progress(
+            SpinnerColumn(style="dim"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=28),
+            MofNCompleteColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+            TimeRemainingColumn(),
+            TextColumn("{task.fields[note]}"),
+            console=self.console,
+            transient=False,
+        )
+        self._progress.start()
+        for candidate_id in plan.matrix.candidate_ids:
+            total = len(plan.matrix.for_candidate(candidate_id))
+            if not total:
+                continue
+            self._tasks[candidate_id] = self._progress.add_task(
+                f"  {candidate_id}", total=total, note=""
+            )
+        self._overall = self._progress.add_task(
+            "  all", total=len(plan.matrix.units), note=""
+        )
+
+    def on_unit_start(self, unit: WorkUnit) -> None:
+        if self._progress is None:
+            return
+        task = self._tasks.get(unit.candidate_id)
+        if task is not None:
+            self._progress.update(task, note=f"[dim]{unit.case_id}#{unit.repeat_idx}[/dim]")
+
+    def on_unit_done(self, result: UnitResult) -> None:
+        self._tally.add(result)
+        if self._progress is None:
+            self._print_line(result)
+            return
+        task = self._tasks.get(result.candidate_id)
+        if task is not None:
+            self._progress.advance(task)
+        if self._overall is not None:
+            self._progress.update(
+                self._overall, advance=1, note=f"[dim]{self._tally.render()}[/dim]"
+            )
+        # A forbidden tool call is not something to leave in a progress field
+        # that scrolls away. It is printed as its own line, in red, while the
+        # run continues.
+        if result.outcome is Outcome.FORBIDDEN_TOOL:
+            self._progress.console.print(
+                Text(
+                    f"  {symbols(self.ascii_mode).fail} FORBIDDEN TOOL in "
+                    f"{result.unit.case_id} ({result.candidate_id})",
+                    style="bold red",
+                )
+            )
+        elif result.outcome is Outcome.HARNESS_ERROR:
+            self._progress.console.print(
+                Text(
+                    f"  {symbols(self.ascii_mode).fail} harness error in "
+                    f"{result.unit}: {result.error_class} - this is our bug, not "
+                    "the model's",
+                    style="bold magenta",
+                )
+            )
+
+    def _print_line(self, result: UnitResult) -> None:
+        """One line per execution, for a log rather than a terminal.
+
+        Failures and anything unsafe always print. Passes do not: a 1,500-row
+        CI log of "ok" is noise that hides the twelve lines that matter.
+        """
+        if result.outcome in _QUIET_OUTCOMES:
+            return
+        sym = symbols(self.ascii_mode)
+        self.console.print(
+            Text(
+                f"  {sym.fail} {result.unit} {sym.arrow} {result.outcome.value}"
+                + (f": {result.error_message}" if result.error_message else ""),
+                style="red" if result.outcome is not Outcome.UNCOVERED else "cyan",
+            )
+        )
+
+    def on_note(self, message: str) -> None:
+        target = self._progress.console if self._progress else self.console
+        target.print(Text(f"  {symbols(self.ascii_mode).skip} {message}", style="dim"))
+
+    def on_finish(self, summary: RunSummary) -> None:
+        self.close()
+        sym = symbols(self.ascii_mode)
+        self.console.print()
+        style = "bold green" if summary.complete else "bold yellow"
+        line = (
+            f"  {sym.ok if summary.complete else sym.warn} "
+            f"{summary.executed} executed"
+        )
+        if summary.resumed:
+            line += f", {summary.resumed} resumed"
+        line += f" of {summary.planned} · {self._tally.render()}"
+        if summary.spend_usd:
+            line += f" · ${summary.spend_usd:.4f}"
+        self.console.print(Text(line, style=style))
+        if summary.aborted_reason:
+            self.console.print(
+                Text(f"  {sym.warn} {summary.aborted_reason}", style="yellow")
+            )
+        self.console.print()
+
+    @property
+    def tally(self) -> Tally:
+        return self._tally
+
+
+__all__ = ["ProgressObserver", "Tally"]

slmeval/report/scorecard.py

@@ -0,0 +1,367 @@
+"""The scorecard. For a CLI tool, the terminal table *is* the product.
+
+This is what gets screenshotted, posted, and pasted into a pull request, so it
+gets the same care as the measurement code. Pure functions from a
+:class:`~slmeval.report.model.Scorecard` to a Rich renderable - nothing here
+prints, which is what makes every view snapshot-testable.
+
+Four rules it follows, all of them about not being misleading:
+
+**Always print ``n``.** A rate without its sample size is a number pretending
+to be evidence, and the whole product's credibility rests on nobody
+over-reading three samples.
+
+**Always print the interval, next to the point estimate.** Seeing ``100%`` and
+``[44%, 100%]`` side by side teaches the reader more about ``n = 3`` than any
+documentation will.
+
+**Colour by delta, not by absolute value.** 83% is not red; 83% where the
+baseline got 97% is. Colouring absolutes would paint a hard benchmark red
+everywhere and a trivial one green, telling the reader nothing either way.
+
+**Warnings go above the table.** A forbidden tool call must never be a column
+somebody scrolls past. It appears before the first number, and so does a
+harness error, because that one calls the whole run into question.
+"""
+
+from __future__ import annotations
+
+from rich.console import Group, RenderableType
+from rich.table import Table
+from rich.text import Text
+
+from slmeval.cli.render import Symbols, ascii_only, symbols
+from slmeval.cli.render import dash as dash_for
+from slmeval.report.model import CandidateReport, Scorecard
+
+GOOD = "green"
+BAD = "red"
+MEH = "yellow"
+DIM = "dim"
+
+
+def render_scorecard(
+    card: Scorecard,
+    *,
+    ascii_mode: bool | None = None,
+    show_nodes: bool = False,
+) -> RenderableType:
+    """The whole scorecard: header, warnings, table, footer."""
+    sym = symbols(ascii_mode)
+    parts: list[RenderableType] = [
+        Text(""),
+        header(card),
+        Text(""),
+    ]
+    notes = warnings_block(card, ascii_mode=ascii_mode)
+    if notes is not None:
+        parts += [notes, Text("")]
+    parts.append(table(card, ascii_mode=ascii_mode))
+    if show_nodes:
+        from slmeval.report.attribution import render_attribution
+
+        parts += [Text(""), render_attribution(card, ascii_mode=ascii_mode)]
+    parts += [Text(""), footer(card, sym)]
+    return Group(*parts)
+
+
+def header(card: Scorecard) -> Text:
+    """One line naming everything that would invalidate a comparison.
+
+    The seed, the tool mode and the fault rate are in the headline rather than
+    a footnote because they are what someone re-running this needs, and because
+    two tables that differ in any of them are not comparable however similar
+    they look.
+    """
+    run = card.run
+    out = Text("  ")
+    out.append(run.name, style="bold")
+    bits = [
+        f"{len(card.cases)} cases x {card.repeats} runs",
+        run.mode,
+        f"faults {run.fault_rate:.0%}" if run.fault_rate else "no faults",
+        f"seed {run.seed}",
+        run.scheduling,
+    ]
+    out.append(" · ", style=DIM)
+    out.append(" · ".join(bits), style=DIM)
+    out.append("\n  ")
+    out.append(f"{run.run_id}", style=DIM)
+    if run.git_sha:
+        out.append(f" · {run.git_sha}", style=DIM)
+    out.append(f" · dataset {run.dataset_hash}", style=DIM)
+    return out
+
+
+def warnings_block(
+    card: Scorecard, *, ascii_mode: bool | None = None
+) -> Text | None:
+    """The lines above the table, worst first."""
+    notes = card.warnings()
+    if not notes:
+        return None
+    sym = symbols(ascii_mode)
+    out = Text()
+    for index, note in enumerate(notes):
+        glyph, style = _note_style(note, sym)
+        out.append(f"  {glyph} ", style=style)
+        out.append(note, style=style)
+        if index < len(notes) - 1:
+            out.append("\n")
+    return out
+
+
+def _note_style(note: str, sym: Symbols) -> tuple[str, str]:
+    lowered = note.lower()
+    if "forbidden" in lowered:
+        return sym.fail, "bold red"
+    if "inside slmeval" in lowered:
+        return sym.fail, "bold magenta"
+    # Not red, and that is the point: an uncovered path is a gap in our
+    # recordings, not a model failure, and colouring it like one would undo the
+    # distinction the diagnosis exists to draw.
+    if "do not cover" in lowered:
+        return sym.warn, "cyan"
+    if "latency not measured" in lowered:
+        return sym.skip, DIM
+    return sym.warn, MEH
+
+
+def table(card: Scorecard, *, ascii_mode: bool | None = None) -> Table:
+    """The table itself."""
+    dash = dash_for(ascii_mode)
+    grid = Table(
+        box=None,
+        pad_edge=False,
+        padding=(0, 2),
+        header_style="bold",
+        show_edge=False,
+    )
+    grid.add_column("Candidate", no_wrap=True)
+    grid.add_column(f"pass^{card.repeats}", justify="right", no_wrap=True)
+    grid.add_column("95% CI", justify="right", no_wrap=True)
+    grid.add_column("runs", justify="right", no_wrap=True)
+    grid.add_column("Schema", justify="right", no_wrap=True)
+    grid.add_column("Tool F1", justify="right", no_wrap=True)
+    grid.add_column("Steps", justify="right", no_wrap=True)
+    if card.latency_reportable:
+        grid.add_column("p95 ms", justify="right", no_wrap=True)
+    grid.add_column("$/case", justify="right", no_wrap=True)
+    grid.add_column(_delta_header(ascii_mode), justify="right", no_wrap=True)
+
+    for row in card.rows:
+        grid.add_row(*_cells(card, row, dash=dash))
+    return grid
+
+
+def _delta_header(ascii_mode: bool | None) -> str:
+    """``d Cost`` on a legacy console, which cannot encode a delta glyph."""
+    return "d Cost" if ascii_only(ascii_mode) else "Δ Cost"
+
+
+def _cells(card: Scorecard, row: CandidateReport, *, dash: str) -> list[RenderableType]:
+    estimated = "~" if row.tokens_estimated else ""
+    cells: list[RenderableType] = [
+        _name(row),
+        _pass_cell(card, row, dash),
+        _interval_cell(row, dash),
+        Text(
+            f"{row.stats.passes}/{row.stats.executions}"
+            if row.stats.executions
+            else dash,
+            style=DIM,
+        ),
+        _rate_cell(card, row, row.schema_valid_rate, "schema", dash),
+        _rate_cell(card, row, row.tool_f1, "f1", dash, digits=2),
+        Text(f"{row.mean_steps:.1f}" if row.mean_steps is not None else dash),
+    ]
+    if card.latency_reportable:
+        cells.append(
+            Text(f"{row.p95_ms:,.0f}" if row.p95_ms is not None else dash)
+        )
+    cells.append(
+        Text(
+            f"{estimated}${row.cost_per_case:.4f}"
+            if row.cost_per_case is not None
+            else dash,
+            style=DIM if row.cost_per_case == 0 else "",
+        )
+    )
+    cells.append(_delta_cost_cell(card, row, dash))
+    return cells
+
+
+def _name(row: CandidateReport) -> Text:
+    out = Text()
+    if row.baseline and row.candidate_id != "baseline":
+        out.append("baseline ", style=DIM)
+    out.append(row.candidate_id, style="bold" if row.baseline else "")
+    return out
+
+
+def _pass_cell(card: Scorecard, row: CandidateReport, dash: str) -> Text:
+    value = row.pass_hat_k
+    if value is None:
+        return Text(dash, style=DIM)
+    delta = card.delta_pass(row)
+    return Text(f"{value:.1%}", style=_delta_style(delta, bigger_is_better=True))
+
+
+def _interval_cell(row: CandidateReport, dash: str) -> Text:
+    interval = row.stats.interval
+    if interval is None:
+        return Text(dash, style=DIM)
+    return Text(interval.render(), style=DIM)
+
+
+def _rate_cell(
+    card: Scorecard,
+    row: CandidateReport,
+    value: float | None,
+    which: str,
+    dash: str,
+    *,
+    digits: int = 0,
+) -> Text:
+    """One rate, coloured by how it compares with the baseline's own.
+
+    ``which`` names the baseline figure to compare against rather than taking
+    the number directly, because the baseline row has to be able to render
+    itself - and it has no delta against itself to colour by.
+    """
+    if value is None:
+        return Text(dash, style=DIM)
+    reference = _reference_value(card, which)
+    delta = None if reference is None or row.baseline else value - reference
+    text = f"{value:.{digits}f}" if digits else f"{value:.0%}"
+    return Text(text, style=_delta_style(delta, bigger_is_better=True))
+
+
+def _delta_cost_cell(card: Scorecard, row: CandidateReport, dash: str) -> Text:
+    delta = card.delta_cost(row)
+    if delta is None:
+        return Text(dash, style=DIM)
+    # Cheaper is better, so the sign convention is inverted against the
+    # quality columns. Spelled out here rather than left to the reader.
+    return Text(f"{delta:+.0%}", style=_delta_style(delta, bigger_is_better=False))
+
+
+def _reference_value(card: Scorecard, which: str) -> float | None:
+    base = card.baseline
+    if base is None:
+        return None
+    return {"schema": base.schema_valid_rate, "f1": base.tool_f1}.get(which)
+
+
+def _delta_style(delta: float | None, *, bigger_is_better: bool) -> str:
+    """Colour from the change, never the level.
+
+    A threshold of one percentage point, so floating-point noise and a genuinely
+    identical result both render neutral instead of flickering green and red
+    between runs of the same config.
+    """
+    if delta is None or abs(delta) < 0.01:
+        return ""
+    improved = delta > 0 if bigger_is_better else delta < 0
+    return GOOD if improved else BAD
+
+
+def footer(card: Scorecard, sym: Symbols) -> Text:
+    """Where the numbers came from, in one dim block.
+
+    The estimator and the price date belong on the artefact, not in the
+    documentation: this table gets screenshotted away from everything that
+    would otherwise explain it.
+    """
+    out = Text()
+    out.append(
+        f"  pass^{card.repeats} = fraction of cases passing every repeat; "
+        "95% CI is the Wilson interval on that fraction\n",
+        style=DIM,
+    )
+    out.append(
+        f"  gate on the lower bound, never the point estimate "
+        f"(n={len(card.cases)} cases)\n",
+        style=DIM,
+    )
+    out.append(f"  prices as of {card.prices_as_of}", style=DIM)
+    if any(r.recovery_rate is not None for r in card.rows):
+        out.append("\n  ", style=DIM)
+        out.append(
+            " · ".join(
+                f"{r.candidate_id} recovered from {r.recovery_rate:.0%} of faults"
+                for r in card.rows
+                if r.recovery_rate is not None
+            ),
+            style=DIM,
+        )
+    return out
+
+
+def render_summary_line(card: Scorecard, *, ascii_mode: bool | None = None) -> Text:
+    """The one-line verdict the whole product exists to print.
+
+        intent_router: gpt-4o -> gemma3:4b - cost -78%, pass^3 96.4% -> 91.2%
+
+    Only for the single-candidate case. With several candidates there is no
+    single recommendation to make, and inventing one would be the tool
+    editorialising over the user's own trade-off.
+    """
+    sym = symbols(ascii_mode)
+    base = card.baseline
+    others = card.candidates
+    if base is None or len(others) != 1:
+        return Text("")
+    row = others[0]
+    cost = card.delta_cost(row)
+    out = Text("  ")
+    out.append(f"{base.model} {sym.arrow} {row.model}", style="bold")
+    if cost is not None:
+        out.append(f" - cost {cost:+.0%}", style=GOOD if cost < 0 else BAD)
+    if base.pass_hat_k is not None and row.pass_hat_k is not None:
+        interval = row.stats.interval
+        tail = f" (CI {interval.render()})" if interval else ""
+        out.append(
+            f", pass^{card.repeats} {base.pass_hat_k:.1%} {sym.arrow} "
+            f"{row.pass_hat_k:.1%}{tail}"
+        )
+    return out
+
+
+def render_rollups(
+    card: Scorecard, *, ascii_mode: bool | None = None, limit: int = 0
+) -> Table:
+    """Per-case detail: which cases each candidate actually failed.
+
+    The scorecard says a candidate lost four points; this says which four cases
+    and how consistently. A user cannot act on the first without the second.
+    """
+    dash = dash_for(ascii_mode)
+    grid = Table(box=None, pad_edge=False, padding=(0, 2), header_style="bold", show_edge=False)
+    grid.add_column("Case", no_wrap=True)
+    for row in card.rows:
+        grid.add_column(row.candidate_id, justify="right", no_wrap=True)
+
+    shown = card.cases[:limit] if limit else card.cases
+    for case_id in shown:
+        cells: list[RenderableType] = [Text(case_id)]
+        for row in card.rows:
+            found = row.rollups.get(case_id)
+            if found is None or not found.n:
+                cells.append(Text(dash, style=DIM))
+                continue
+            style = GOOD if found.all_passed else (MEH if found.c else BAD)
+            cells.append(Text(f"{found.c}/{found.n}", style=style))
+        grid.add_row(*cells)
+    return grid
+
+
+__all__ = [
+    "footer",
+    "header",
+    "render_rollups",
+    "render_scorecard",
+    "render_summary_line",
+    "table",
+    "warnings_block",
+]