slmeval-cli 0.1.0__py3-none-any.whl

slmeval/cli/commands/baseline_cmd.py

@@ -0,0 +1,201 @@
+"""``slmeval baseline`` - write and inspect the committed reference.
+
+``slmeval.baseline.json`` is the file ``slmeval gate --max-regression`` compares
+against, and it belongs in the repository. That is the whole point:
+gating against "the last run in the local database" gives different verdicts on
+different machines and cannot be reviewed, while a committed file turns "we
+accepted a two-point drop" into a diff somebody approved.
+
+So this command is deliberately boring. It takes a stored run, writes the
+metrics and the provenance to a JSON file, and tells the user to commit it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from slmeval.cli.loading import LATEST, open_project, resolve_run
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.errors import ConfigError
+from slmeval.report.baseline import Baseline, baseline_path
+from slmeval.report.model import build_scorecard
+
+baseline_app = typer.Typer(
+    no_args_is_help=True,
+    help="The committed reference a gate compares against.",
+)
+
+_ConfigOpt = Annotated[
+    Path | None,
+    typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
+]
+_OutOpt = Annotated[
+    Path | None,
+    typer.Option("--out", "-o", help="Where to write it. Defaults to slmeval.baseline.json."),
+]
+
+
+@baseline_app.command("save")
+@handle_errors
+def baseline_save(
+    config: _ConfigOpt = None,
+    run_id: Annotated[
+        str, typer.Option("--run-id", help="A run id or prefix, or 'latest'.")
+    ] = LATEST,
+    out_path: _OutOpt = None,
+    force: Annotated[
+        bool,
+        typer.Option("--force", help="Overwrite an existing baseline file."),
+    ] = False,
+) -> None:
+    """Write a stored run's metrics to the committed baseline file."""
+    out = console()
+    sym = symbols()
+
+    with open_project(config, load_agent=False) as project:
+        resolved = resolve_run(project.store, run_id)
+        card = build_scorecard(
+            project.store, resolved, prices_as_of=project.prices.as_of
+        )
+        path = baseline_path(project.root, out_path)
+        previous = Baseline.load(path) if path.is_file() else None
+        if previous is not None and not force:
+            raise ConfigError(
+                f"{path.name} already holds the baseline from run {previous.run_id}",
+                source=path,
+                hint=(
+                    "pass --force to replace it - and commit the diff, because "
+                    "moving a baseline is a decision somebody should review"
+                ),
+            )
+
+        baseline = Baseline.from_scorecard(card)
+        baseline.save(path)
+
+        out.print()
+        out.print(Text(f"  {sym.ok} wrote {path.name}", style="bold green"))
+        out.print()
+        out.print(_table(baseline, previous))
+        out.print()
+        out.print(
+            Text(
+                f"  {sym.arrow} commit it: a gate that compares against an "
+                "uncommitted file gives a different answer on every machine",
+                style="cyan",
+            )
+        )
+        if not card.latency_reportable:
+            out.print(
+                Text(
+                    f"  {sym.skip} no p95 recorded - this run was scheduled for "
+                    "throughput, so its latency describes the scheduler. Re-run "
+                    "with --serial-latency to gate on p95.",
+                    style="dim",
+                )
+            )
+        out.print()
+
+
+@baseline_app.command("show")
+@handle_errors
+def baseline_show(
+    config: _ConfigOpt = None,
+    path_opt: _OutOpt = None,
+) -> None:
+    """Print the committed baseline, so a failing gate can be read against it."""
+    out = console()
+    with open_project(config, load_agent=False) as project:
+        path = baseline_path(project.root, path_opt)
+        baseline = Baseline.load(path)
+
+    out.print()
+    out.print(
+        Text(
+            f"  {path.name}  {baseline.run_id} · saved {baseline.saved_at[:19]} · "
+            f"slmeval {baseline.slmeval_version}",
+            style="dim",
+        )
+    )
+    out.print(
+        Text(
+            f"  dataset {baseline.dataset_hash} · config {baseline.config_hash} · "
+            f"{baseline.mode} · seed {baseline.seed} · "
+            f"{baseline.runs_per_case} runs/case",
+            style="dim",
+        )
+    )
+    out.print()
+    out.print(_table(baseline, None))
+    out.print()
+
+
+def _table(baseline: Baseline, previous: Baseline | None) -> Table:
+    """The rows, with the previous file's numbers beside them when replacing.
+
+    Shown side by side on a ``--force`` overwrite because that is the moment a
+    user is deciding whether to accept a change, and the diff they are about to
+    commit is the evidence.
+    """
+    grid = Table(box=None, pad_edge=False, padding=(0, 2), header_style="bold", show_edge=False)
+    grid.add_column("Candidate", no_wrap=True)
+    grid.add_column("pass^k", justify="right", no_wrap=True)
+    grid.add_column("wilson lo", justify="right", no_wrap=True)
+    grid.add_column("cases", justify="right", no_wrap=True)
+    grid.add_column("$/case", justify="right", no_wrap=True)
+    if previous is not None:
+        grid.add_column("was", justify="right", no_wrap=True)
+
+    for candidate_id, row in sorted(
+        baseline.candidates.items(), key=lambda kv: (kv[1].role != "baseline", kv[0])
+    ):
+        cells = [
+            Text(candidate_id, style="bold" if row.role == "baseline" else ""),
+            Text(_pct(row.pass_hat_k)),
+            Text(_pct(row.wilson_lo), style="dim"),
+            Text(f"{row.cases_all_pass}/{row.cases}", style="dim"),
+            Text(
+                f"${row.cost_per_case:.4f}" if row.cost_per_case is not None else "-",
+                style="dim",
+            ),
+        ]
+        if previous is not None:
+            before = previous.candidate(candidate_id)
+            cells.append(
+                Text(
+                    _pct(before.pass_hat_k) if before else "new",
+                    style=_delta_style(
+                        before.pass_hat_k if before else None, row.pass_hat_k
+                    ),
+                )
+            )
+        grid.add_row(*cells)
+    return grid
+
+
+def _pct(value: float | None) -> str:
+    return f"{value:.1%}" if value is not None else "-"
+
+
+def _delta_style(before: float | None, now: float | None) -> str:
+    """Colour the old number by which way the new one moved.
+
+    One percentage point of slack, same as the scorecard: floating-point noise
+    must not flicker a committed baseline green and red between saves of the
+    same run.
+    """
+    if before is None or now is None or abs(now - before) < 0.01:
+        return "dim"
+    return "green" if now > before else "red"
+
+
+def register(app: typer.Typer) -> None:
+    app.add_typer(baseline_app, name="baseline")
+
+
+__all__ = ["baseline_app", "register"]

slmeval/cli/commands/cache_cmd.py

@@ -0,0 +1,249 @@
+"""`slmeval cache` — what the trace store holds, and how to reclaim it.
+
+Benchmark traces grow to gigabytes. Give users the broom before they go looking
+for one, because the alternative is someone deleting ``.slmeval/`` by hand and
+losing the baselines their gate compares against.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.config.loader import CACHE_DIR_NAME, DEFAULT_CONFIG_NAME
+from slmeval.errors import ConfigError
+from slmeval.storage import RunStore
+
+cache_app = typer.Typer(
+    no_args_is_help=True,
+    help="Trace store statistics and cleanup.",
+)
+
+_AGE_RE = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$", re.IGNORECASE)
+_UNITS = {
+    "s": timedelta(seconds=1),
+    "m": timedelta(minutes=1),
+    "h": timedelta(hours=1),
+    "d": timedelta(days=1),
+    "w": timedelta(weeks=1),
+}
+
+
+def parse_age(text: str) -> timedelta:
+    """``30d``, ``12h``, ``2w``. No bare numbers.
+
+    A bare ``30`` could mean seconds, days or runs, and the command it feeds
+    deletes things. Refusing to guess costs the user one retyped character; a
+    wrong guess costs them their benchmark history.
+    """
+    match = _AGE_RE.match(text)
+    if not match:
+        raise ConfigError(
+            f"cannot read {text!r} as an age",
+            hint="use a number and a unit: 30d, 12h, 2w",
+        )
+    return int(match.group(1)) * _UNITS[match.group(2).lower()]
+
+
+def human_bytes(size: int) -> str:
+    value = float(size)
+    for unit in ("B", "KB", "MB", "GB"):
+        if value < 1024 or unit == "GB":
+            return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}"
+        value /= 1024
+    return f"{value:.1f} GB"
+
+
+def project_root(explicit: Path | None) -> Path:
+    """The directory whose ``.slmeval`` we operate on.
+
+    Walks up for ``eval.toml`` like the other commands, but never parses it:
+    the whole point of ``cache stats`` is to work when something else is broken,
+    and a config too invalid to load must not stop a user inspecting their own
+    disk usage.
+    """
+    if explicit is not None:
+        return Path(explicit).expanduser().resolve()
+    here = Path.cwd().resolve()
+    for directory in (here, *here.parents):
+        if (directory / DEFAULT_CONFIG_NAME).is_file() or (directory / CACHE_DIR_NAME).is_dir():
+            return directory
+    raise ConfigError(
+        f"no {DEFAULT_CONFIG_NAME} or {CACHE_DIR_NAME}/ found in {here} or any parent",
+        hint="run this from a project directory, or pass --path",
+    )
+
+
+_PathOpt = Annotated[
+    Path | None,
+    typer.Option("--path", help="Project directory to inspect. Defaults to the one you are in."),
+]
+
+
+@cache_app.command("stats")
+@handle_errors
+def cache_stats(
+    path: _PathOpt = None,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Also show the schema migration history.")
+    ] = False,
+) -> None:
+    """Show what the trace store holds and what it costs on disk."""
+    root = project_root(path)
+    out = console()
+    sym = symbols()
+
+    with RunStore.for_project(root) as store:
+        stats = store.stats()
+        cassettes = root / CACHE_DIR_NAME / "cassettes.sqlite"
+        cassette_bytes = cassettes.stat().st_size if cassettes.is_file() else 0
+
+        table = Table.grid(padding=(0, 2))
+        table.add_column(style="dim", no_wrap=True)
+        table.add_column(overflow="fold")
+        table.add_row("store", str(store.path))
+        table.add_row("schema", f"v{store.schema_version}")
+        table.add_row("runs", str(stats.runs))
+        if stats.running:
+            table.add_row("in progress", Text(str(stats.running), style="yellow"))
+        table.add_row("executions", str(stats.executions))
+        table.add_row("model calls", str(stats.node_traces))
+        table.add_row("tool calls", str(stats.tool_traces))
+        table.add_row("evaluations", str(stats.evaluations))
+        if stats.oldest_run:
+            table.add_row("oldest run", stats.oldest_run[:19])
+            table.add_row("newest run", (stats.newest_run or "")[:19])
+        table.add_row("database", human_bytes(stats.db_bytes))
+        table.add_row("payloads", f"{stats.blobs} files, {human_bytes(stats.blob_bytes)}")
+        if cassette_bytes:
+            table.add_row("cassettes", human_bytes(cassette_bytes))
+        table.add_row(
+            "total",
+            human_bytes(stats.db_bytes + stats.blob_bytes + cassette_bytes),
+        )
+
+        out.print()
+        out.print(table)
+        if verbose:
+            out.print()
+            out.print(Text("  migrations", style="dim"))
+            for version, name, at, built_by in store.migration_history():
+                out.print(
+                    Text(
+                        f"    v{version} {name} · {at[:19]} · slmeval {built_by}",
+                        style="dim",
+                    )
+                )
+        out.print()
+        if stats.runs == 0:
+            out.print(Text(f"  {sym.skip} no runs stored yet", style="dim"))
+            out.print()
+
+
+@cache_app.command("prune")
+@handle_errors
+def cache_prune(
+    older_than: Annotated[
+        str,
+        typer.Option("--older-than", help="Delete runs older than this: 30d, 12h, 2w."),
+    ] = "30d",
+    keep_last: Annotated[
+        int,
+        typer.Option(
+            "--keep-last",
+            min=0,
+            help="Always keep this many of the most recent runs, whatever their age.",
+        ),
+    ] = 3,
+    path: _PathOpt = None,
+    dry_run: Annotated[
+        bool, typer.Option("--dry-run", help="Show what would go, delete nothing.")
+    ] = False,
+    yes: Annotated[bool, typer.Option("--yes", "-y", help="Skip the confirmation.")] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", help="Prune even with runs marked in progress (crash leftovers)."),
+    ] = False,
+) -> None:
+    """Delete old runs and the payloads nothing references any more."""
+    root = project_root(path)
+    age = parse_age(older_than)
+    out = console()
+    sym = symbols()
+
+    with RunStore.for_project(root) as store:
+        cutoff = datetime.now(UTC) - age
+        stale = store.runs_older_than(cutoff, keep_last=keep_last)
+
+        out.print()
+        if not stale:
+            out.print(Text(f"  {sym.ok} nothing older than {older_than} to prune", style="green"))
+            out.print()
+            # Payloads can still be orphaned by a crash between a blob write
+            # and its commit, so sweep anyway - just quietly.
+            swept = store.blobs.sweep(store.referenced_blobs(), dry_run=dry_run)
+            if swept.deleted:
+                out.print(
+                    Text(
+                        f"  {sym.ok} freed {human_bytes(swept.freed_bytes)} "
+                        f"of unreferenced payloads",
+                        style="green",
+                    )
+                )
+                out.print()
+            return
+
+        table = Table.grid(padding=(0, 2))
+        table.add_column(style="dim", no_wrap=True)
+        table.add_column(overflow="fold")
+        for run in stale[:10]:
+            table.add_row(run.run_id, f"{run.name} · {run.started_at[:19]}")
+        if len(stale) > 10:
+            table.add_row("", Text(f"... and {len(stale) - 10} more", style="dim"))
+        out.print(Text(f"  {len(stale)} run(s) older than {older_than}:", style="bold"))
+        out.print()
+        out.print(table)
+        out.print()
+
+        if dry_run:
+            out.print(Text(f"  {sym.skip} dry run, nothing deleted", style="dim"))
+            out.print()
+            return
+        # Deleting benchmark history is not undoable, and the thing being
+        # deleted is what the next gate compares against. Ask.
+        if not yes and not typer.confirm("  Delete these runs and their traces?"):
+            out.print()
+            out.print(Text(f"  {sym.skip} cancelled", style="dim"))
+            out.print()
+            raise typer.Exit()
+
+        deleted, swept = store.prune(older_than=age, keep_last=keep_last, force=force)
+        out.print()
+        out.print(
+            Text(
+                f"  {sym.ok} deleted {len(deleted)} run(s), freed "
+                f"{human_bytes(swept.freed_bytes)} of payloads",
+                style="bold green",
+            )
+        )
+        if swept.kept_recent:
+            out.print(
+                Text(
+                    f"    kept {swept.kept_recent} recent unreferenced payload(s) - "
+                    "a sweep spares anything a run may still be committing",
+                    style="dim",
+                )
+            )
+        out.print()
+
+
+def register(app: typer.Typer) -> None:
+    app.add_typer(cache_app, name="cache")

slmeval/cli/commands/dataset_cmd.py

@@ -0,0 +1,52 @@
+"""`slmeval dataset` — work with golden case files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.dataset import load_dataset
+
+dataset_app = typer.Typer(
+    no_args_is_help=True,
+    help="Inspect and validate golden datasets.",
+)
+
+
+@dataset_app.command("validate")
+@handle_errors
+def validate_dataset(
+    path: Annotated[Path, typer.Argument(help="Path to a .jsonl dataset.")],
+) -> None:
+    """Check a dataset file: JSON, schema, unique ids, compilable regexes."""
+    out = console()
+    sym = symbols()
+    dataset = load_dataset(path)
+
+    table = Table.grid(padding=(0, 2))
+    table.add_column(style="dim", no_wrap=True)
+    table.add_column(overflow="fold")
+    table.add_row("file", str(dataset.path))
+    table.add_row("cases", str(len(dataset)))
+    table.add_row("hash", dataset.content_hash)
+    if dataset.tags:
+        table.add_row("tags", ", ".join(dataset.tags))
+
+    safety = sum(1 for c in dataset.cases if c.assertions.forbidden_tools)
+    table.add_row("safety cases", f"{safety} declare forbidden_tools")
+
+    out.print()
+    out.print(table)
+    out.print()
+    out.print(Text(f"  {sym.ok} valid", style="bold green"))
+    out.print()
+
+
+def register(app: typer.Typer) -> None:
+    app.add_typer(dataset_app, name="dataset")

slmeval/cli/commands/doctor_cmd.py

@@ -0,0 +1,68 @@
+"""`slmeval doctor` — can this machine run the benchmark right now?"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.text import Text
+
+from slmeval import exit_codes
+from slmeval.cli.render import checks_summary, console, render_checks
+from slmeval.cli.wrap import handle_errors
+from slmeval.config import LoadedConfig, find_config, load_config
+from slmeval.doctor import Status, run_checks, worst_status
+from slmeval.errors import SlmevalError
+
+
+def _try_load(config: Path | None) -> tuple[LoadedConfig | None, str | None, Path]:
+    """Load the config if there is one, reporting failure as a check rather than
+    an exception — doctor's whole job is to report problems, not to hit them."""
+    try:
+        path = find_config(config)
+    except SlmevalError as err:
+        return None, (err.message if config is not None else None), Path.cwd()
+
+    try:
+        return load_config(path), None, path.parent
+    except SlmevalError as err:
+        return None, err.message, path.parent
+
+
+def register(app: typer.Typer) -> None:
+    @app.command()
+    @handle_errors
+    def doctor(
+        config: Annotated[
+            Path | None,
+            typer.Option("--config", "-c", help="Path to eval.toml. Default: search upwards."),
+        ] = None,
+        skip_network: Annotated[
+            bool,
+            typer.Option("--skip-network", help="Do not probe providers (offline / CI use)."),
+        ] = False,
+    ) -> None:
+        """Check Python, storage, credentials and provider reachability."""
+        out = console()
+        loaded, config_error, root = _try_load(config)
+
+        checks = run_checks(
+            root=root,
+            loaded=loaded,
+            config_error=config_error,
+            skip_network=skip_network,
+        )
+
+        out.print()
+        out.print(render_checks(checks))
+        out.print()
+        out.print(Text("  ", end=""), end="")
+        out.print(checks_summary(checks))
+        out.print()
+
+        status = worst_status(checks)
+        if status is Status.FAIL:
+            # Exit 3, never 1. A broken environment is not a model regression,
+            # and CI must be able to tell them apart.
+            raise typer.Exit(exit_codes.ENVIRONMENT)