slmeval-cli 0.1.0__py3-none-any.whl

slmeval/cli/commands/init_cmd.py

@@ -0,0 +1,144 @@
+"""`slmeval init` — scaffold a benchmark in the current project."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated, Literal
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.config import templates
+from slmeval.errors import SlmevalError
+
+Outcome = Literal["created", "skipped", "appended"]
+
+
+@dataclass
+class Written:
+    path: Path
+    outcome: Outcome
+    note: str = ""
+
+
+def slugify(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or "agent"
+
+
+def default_benchmark_name(directory: Path) -> str:
+    return f"{slugify(directory.resolve().name)}-slm-downsizing"
+
+
+def scaffold(directory: Path, *, name: str | None = None, force: bool = False) -> list[Written]:
+    """Write the starter files. Never overwrites without ``force``."""
+    directory = Path(directory)
+    if directory.exists() and not directory.is_dir():
+        raise SlmevalError(f"{directory} exists and is not a directory")
+    directory.mkdir(parents=True, exist_ok=True)
+
+    benchmark_name = name or default_benchmark_name(directory)
+    plan: list[tuple[Path, str]] = [
+        (directory / "eval.toml", templates.eval_toml(benchmark_name)),
+        (directory / "datasets" / "golden_cases.jsonl", templates.DATASET_JSONL),
+        (directory / "slmeval_target.py", templates.TARGET_PY),
+    ]
+
+    written: list[Written] = []
+    for path, content in plan:
+        if path.exists() and not force:
+            written.append(Written(path, "skipped", "already exists; --force overwrites"))
+            continue
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # newline="\n" so the file hashes identically on Windows and in CI.
+        path.write_text(content, encoding="utf-8", newline="\n")
+        written.append(Written(path, "created"))
+
+    written.append(_ensure_gitignore(directory))
+    return written
+
+
+def _ensure_gitignore(directory: Path) -> Written:
+    """Keep the trace cache out of git without clobbering an existing file."""
+    path = directory / ".gitignore"
+    entry = templates.GITIGNORE_ENTRY
+    if not path.exists():
+        path.write_text(entry + "\n", encoding="utf-8", newline="\n")
+        return Written(path, "created")
+
+    existing = path.read_text(encoding="utf-8-sig").splitlines()
+    if any(line.strip().rstrip("/") == entry.rstrip("/") for line in existing):
+        return Written(path, "skipped", f"already ignores {entry}")
+
+    with path.open("a", encoding="utf-8", newline="\n") as fh:
+        if existing and existing[-1].strip():
+            fh.write("\n")
+        fh.write(f"# slmeval trace cache\n{entry}\n")
+    return Written(path, "appended", f"added {entry}")
+
+
+def register(app: typer.Typer) -> None:
+    @app.command()
+    @handle_errors
+    def init(
+        directory: Annotated[
+            Path,
+            typer.Argument(help="Where to scaffold. Defaults to the current directory."),
+        ] = Path(),
+        name: Annotated[
+            str | None,
+            typer.Option("--name", "-n", help="Benchmark name written into eval.toml."),
+        ] = None,
+        force: Annotated[
+            bool,
+            typer.Option("--force", help="Overwrite files that already exist."),
+        ] = False,
+    ) -> None:
+        """Create eval.toml, a starter dataset, and a target stub."""
+        out = console()
+        sym = symbols()
+        written = scaffold(directory, name=name, force=force)
+
+        table = Table.grid(padding=(0, 2))
+        table.add_column(width=9)
+        table.add_column(style="bold", no_wrap=True)
+        table.add_column(style="dim", overflow="fold")
+        for item in written:
+            style = {"created": "green", "appended": "green", "skipped": "yellow"}[item.outcome]
+            table.add_row(
+                Text(item.outcome, style=style),
+                str(item.path),
+                item.note,
+            )
+
+        out.print()
+        out.print(table)
+        out.print()
+
+        if any(w.outcome == "skipped" and "already exists" in w.note for w in written):
+            out.print(
+                Text(
+                    f"  {sym.warn} some files were left alone - pass --force to overwrite",
+                    style="yellow",
+                )
+            )
+            out.print()
+
+        out.print(Text("  Next:", style="bold"))
+        out.print(Text("    slmeval doctor      ", style="cyan"), end="")
+        out.print(Text("check this machine can run the benchmark", style="dim"))
+        out.print(Text("    slmeval validate    ", style="cyan"), end="")
+        out.print(Text("check eval.toml and the dataset", style="dim"))
+        out.print()
+        out.print(
+            Text(
+                "  Then point [target].entrypoint at your own agent and edit the dataset.",
+                style="dim",
+            )
+        )
+        out.print()

slmeval/cli/commands/judge_cmd.py

@@ -0,0 +1,284 @@
+"""``slmeval judge`` - measure the judge before trusting it.
+
+An LLM judge is the only evaluator here that costs money and the only one that
+can be confidently wrong, so it does not get to produce a number until this
+command has said what its agreement with human labels is. That is the whole
+reason `judge calibrate` exists before Tier 3 is wired into a run: a judge
+whose agreement nobody has measured is a random number generator with a
+confident tone.
+
+The output is three things, in this order, because that is the order a reader
+needs them:
+
+1. the raw agreement **with its Wilson interval**, because thirty labels is a
+   small sample and a point estimate from one is exactly the mistake the rest
+   of this tool exists to avoid;
+2. Cohen's **kappa** beside it, because on a set where nine pairs in ten pass,
+   a judge that says "pass" to everything scores 90% and knows nothing;
+3. the **disagreements**, listed. The deliverable of a calibration run is not
+   a score, it is the handful of pairs where the judge and the human parted
+   company - that is what tells you whether the rubric is ambiguous, the judge
+   is lenient, or the human was wrong.
+
+It writes `slmeval.judge.json`, which is committed, for the same reason the
+baseline file is: "we accepted a judge that agrees with us 71% of the time"
+should be a diff somebody approved.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sqlite3
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from slmeval import exit_codes
+from slmeval.cli.loading import open_project
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.config.models import JudgeCfg
+from slmeval.errors import ConfigError
+from slmeval.eval.calibration import (
+    MIN_LABELS,
+    Agreement,
+    Label,
+    agreement,
+    calibration_path,
+    load_labels,
+    verdict_counts,
+)
+from slmeval.eval.tier3 import Judge, Judgement, JudgementCache
+from slmeval.providers.registry import build_client
+
+judge_app = typer.Typer(
+    no_args_is_help=True,
+    help="Measure and inspect the Tier 3 judge.",
+)
+
+_ConfigOpt = Annotated[
+    Path | None,
+    typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
+]
+
+
+@judge_app.command("calibrate")
+@handle_errors
+def judge_calibrate(
+    labels_path: Annotated[
+        Path,
+        typer.Argument(
+            metavar="LABELS",
+            help="A JSONL file of human-labelled pairs: case_id, rubric, "
+            "reference, candidate, verdict.",
+        ),
+    ],
+    config: _ConfigOpt = None,
+    out_path: Annotated[
+        Path | None,
+        typer.Option("--out", "-o", help="Where to write it. Default: slmeval.judge.json."),
+    ] = None,
+    min_agreement: Annotated[
+        float | None,
+        typer.Option(
+            "--min-agreement",
+            min=0.0,
+            max=1.0,
+            help="Exit 1 if the Wilson lower bound falls below this. "
+            "Defaults to [judge].min_agreement.",
+        ),
+    ] = None,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Emit the record as JSON instead of a table.")
+    ] = False,
+    ascii_mode: Annotated[
+        bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
+    ] = False,
+) -> None:
+    """Run the judge over human-labelled pairs and report its agreement."""
+    out = console()
+    sym = symbols(ascii_mode or None)
+
+    labels = load_labels(labels_path)
+
+    with open_project(config, load_agent=False) as project:
+        cfg = project.config.judge
+        if not cfg.enabled:
+            raise ConfigError(
+                "no judge model configured",
+                hint=(
+                    'Add a [judge] section to eval.toml: provider = "openai", '
+                    'model = "gpt-4o-mini". Calibration needs the model it is '
+                    "about to measure."
+                ),
+            )
+        floor = min_agreement if min_agreement is not None else cfg.min_agreement
+        result = asyncio.run(_calibrate(cfg, labels, project.store.connection()))
+        path = calibration_path(project.root, out_path)
+        path.write_text(
+            json.dumps(result.payload(), indent=2) + "\n", encoding="utf-8"
+        )
+
+    if as_json:
+        print(json.dumps(result.payload(), indent=2))
+    else:
+        out.print()
+        out.print(_summary(result, labels, sym.arrow))
+        out.print()
+        out.print(_confusion(result))
+        if result.disagreements:
+            out.print()
+            out.print(_disagreements(result))
+        for note in result.warnings():
+            out.print(Text(f"  {sym.warn} {note}", style="yellow"))
+        out.print()
+        out.print(Text(f"  {sym.ok} wrote {path.name} - commit it", style="dim"))
+        out.print()
+
+    lower = result.lower
+    if not result.usable:
+        # Not a gate failure: nothing was measured, so there is nothing to
+        # fail. Exit 2 says "this invocation did not answer the question",
+        # which is a usage problem and not a verdict about the judge.
+        raise typer.Exit(exit_codes.USAGE)
+    if lower is not None and lower < floor:
+        out.print(
+            Text(
+                f"  {sym.fail} agreement lower bound {lower:.0%} is below the "
+                f"{floor:.0%} floor - this judge is not ready to grade anything",
+                style="bold red",
+            )
+        )
+        out.print()
+        raise typer.Exit(exit_codes.GATE_FAILED)
+
+
+async def _calibrate(
+    cfg: JudgeCfg, labels: list[Label], conn: sqlite3.Connection
+) -> Agreement:
+    """Judge every labelled pair, then compare.
+
+    Cached like any other judgement, so re-running a calibration after fixing
+    one label costs one call rather than thirty. ``uncalibrated_ok`` is true
+    here by definition: this is the run that produces the calibration.
+    """
+    client = build_client(cfg)
+    judge = Judge(client=client, cache=JudgementCache(conn), uncalibrated_ok=True)
+    pairs: list[tuple[Label, Judgement]] = []
+    try:
+        for label in labels:
+            judgement = await judge.judge(
+                rubric=label.rubric,
+                reference=label.reference,
+                candidate=label.candidate,
+                case_id=label.case_id,
+                question=label.question,
+            )
+            pairs.append((label, judgement))
+    finally:
+        await client.aclose()
+    return agreement(pairs, judge_model=judge.model)
+
+
+# --------------------------------------------------------------------------- #
+# Rendering
+# --------------------------------------------------------------------------- #
+
+
+def _summary(result: Agreement, labels: list[Label], arrow: str) -> Table:
+    grid = Table.grid(padding=(0, 2))
+    grid.add_column(style="dim", no_wrap=True)
+    grid.add_column(overflow="fold")
+
+    grid.add_row("judge", result.judge_model)
+    grid.add_row("labels", f"{result.n} ({_counts(verdict_counts(labels))})")
+    rate = f"{result.rate:.1%}" if result.rate is not None else "-"
+    interval = result.interval.render() if result.interval else "-"
+    grid.add_row("agreement", f"{rate}  {interval}")
+    grid.add_row(
+        "kappa",
+        f"{result.kappa:.2f}" if result.kappa is not None else "undefined",
+    )
+    grid.add_row("verdict", _verdict_line(result))
+    grid.add_row(
+        "",
+        f"{arrow} gate on the lower bound, not the rate: {result.n} labels is a "
+        "small sample and the interval says how small",
+    )
+    return grid
+
+
+def _verdict_line(result: Agreement) -> str:
+    """Whether this is a calibration, and if not, which of the two reasons.
+
+    Naming the reason matters: "you need more labels" and "these are not your
+    labels" are fixed by completely different work, and a single message for
+    both sends half the readers to the wrong one.
+    """
+    if result.usable:
+        return "usable as a calibration"
+    if result.illustrative:
+        return "NOT a calibration - these labels are not human labels"
+    return f"NOT a calibration - {MIN_LABELS} human labels is the floor, this has {result.n}"
+
+
+def _confusion(result: Agreement) -> Table:
+    """Human down the side, judge across the top.
+
+    Worth printing even at n=30: the shape says *how* a judge is wrong, and
+    "lenient" and "harsh" need opposite fixes.
+    """
+    labels = ("pass", "fail", "unclear")
+    table = Table(
+        title="human (rows) vs judge (columns)",
+        title_style="dim",
+        box=None,
+        pad_edge=False,
+    )
+    table.add_column("", style="dim")
+    for name in labels:
+        table.add_column(name, justify="right")
+    for human in labels:
+        cells = [
+            str(result.confusion.get((human, judge), 0)) for judge in labels
+        ]
+        table.add_row(human, *cells)
+    return table
+
+
+def _disagreements(result: Agreement) -> Table:
+    table = Table(
+        title="where the judge and the human parted company",
+        title_style="dim",
+        box=None,
+        pad_edge=False,
+    )
+    table.add_column("case", style="cyan", no_wrap=True)
+    table.add_column("human", no_wrap=True)
+    table.add_column("judge", no_wrap=True)
+    table.add_column("score", justify="right", no_wrap=True)
+    table.add_column("the judge's reason", overflow="fold")
+    for item in result.disagreements:
+        table.add_row(
+            item.case_id,
+            item.human,
+            item.judge,
+            f"{item.score:.2f}",
+            item.reasoning,
+        )
+    return table
+
+
+def _counts(counts: dict[str, int]) -> str:
+    return ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) or "none"
+
+
+def register(app: typer.Typer) -> None:
+    app.add_typer(judge_app, name="judge")
+
+
+__all__ = ["judge_app", "judge_calibrate", "register"]

slmeval/cli/commands/record_cmd.py

@@ -0,0 +1,154 @@
+"""``slmeval record`` - execute the real tools once, so every later run is free.
+
+The matrix runner does the work; this is a narrow front door onto it with three
+settings fixed, and each of them prevents a specific way recording goes wrong.
+
+**The baseline only.** Cassettes are what candidates replay, so they have to be
+recorded by the model you trust. Letting a 4B candidate record them would bake
+its mistakes into the reference every other candidate is measured against.
+
+**One repeat per case.** Cassettes are scoped per case, not per repeat - every
+repeat of a case must replay the same tool responses, or the repeats measure
+different things. Recording three times would just overwrite the same rows
+twice while tripling the calls to someone's real API.
+
+**Serially.** Two executions recording the same case into one store at once is
+a write race, and the loser's payloads simply are not there later - surfacing
+as a cassette miss, which reads as a model failure.
+
+``--fill-gaps`` switches to ``auto``: replay what is recorded, record what is
+not. That is what to run after adding a case, and it is the one mode where an
+existing recording is never touched.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.text import Text
+
+from slmeval import exit_codes
+from slmeval.cli.loading import open_project
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.core.runner import MatrixRunner, RunOptions
+from slmeval.eval.pipeline import evaluator_for
+from slmeval.report.progress import ProgressObserver
+from slmeval.storage.models import Outcome
+
+
+@handle_errors
+def record(
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
+    ] = None,
+    case: Annotated[
+        list[str] | None,
+        typer.Option("--case", help="Only these case ids."),
+    ] = None,
+    fill_gaps: Annotated[
+        bool,
+        typer.Option(
+            "--fill-gaps",
+            help="Replay what is already recorded and record only what is missing. "
+            "Run this after adding a case.",
+        ),
+    ] = False,
+    ascii_mode: Annotated[
+        bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
+    ] = False,
+    quiet: Annotated[bool, typer.Option("--quiet", "-q", help="No progress bars.")] = False,
+) -> None:
+    """Execute real tools once and cache them for replay."""
+    out = console()
+    sym = symbols(ascii_mode or None)
+
+    with open_project(config) as project:
+        options = RunOptions.from_config(
+            project.config,
+            repeats=1,
+            tool_mode="auto" if fill_gaps else "record",
+            # Recording a faulted payload would poison every later replay of
+            # that call, and the poison would be indistinguishable from a real
+            # recording. The sandbox refuses to inject while recording; this
+            # makes the intent explicit at the front door too.
+            fault_rate=0.0,
+            candidate_ids=(project.config.baseline.id,),
+            case_ids=tuple(case) if case else None,
+        )
+        runner = MatrixRunner(
+            config=project.config,
+            dataset=project.dataset,
+            target=project.target,
+            store=project.store,
+            options=options,
+            cassettes=project.cassettes,
+            evaluator=evaluator_for(project.config),
+            prices=project.prices,
+            root=project.root,
+            name=f"{project.config.benchmark.name} (record)",
+        )
+        observer = ProgressObserver(console=out, ascii_mode=ascii_mode or None, quiet=quiet)
+        runner.observer = observer
+        try:
+            summary = asyncio.run(runner.run())
+        except KeyboardInterrupt:
+            observer.close()
+            out.print()
+            out.print(
+                Text(
+                    f"  {sym.warn} interrupted - cassettes recorded so far are kept",
+                    style="yellow",
+                )
+            )
+            out.print()
+            raise typer.Exit(exit_codes.USAGE) from None
+
+        stats = project.cassettes.stats()
+        failed = [r for r in summary.results if r.outcome is not Outcome.PASSED]
+        out.print(
+            Text(
+                f"  {sym.ok} recorded {stats.cassettes} cassette(s) across "
+                f"{len(summary.results)} execution(s)",
+                style="bold green",
+            )
+        )
+        out.print(
+            Text(
+                "    every later run can now use --tool-mode replay: no network, "
+                "no API keys, no GPU",
+                style="dim",
+            )
+        )
+        if failed:
+            # A case that could not be recorded has no coverage, so every
+            # candidate will miss on it and read as a model failure. Exit 4 -
+            # insufficient coverage - rather than letting that happen quietly.
+            out.print()
+            for result in failed:
+                out.print(
+                    Text(
+                        f"  {sym.fail} {result.unit.case_id}: {result.outcome.value}"
+                        + (f" - {result.error_message}" if result.error_message else ""),
+                        style="red",
+                    )
+                )
+            out.print()
+            out.print(
+                Text(
+                    f"  {len(failed)} case(s) were not recorded, so candidates will "
+                    "miss on them. Fix those before benchmarking.",
+                    style="yellow",
+                )
+            )
+            out.print()
+            raise typer.Exit(exit_codes.COVERAGE)
+        out.print()
+
+
+def register(app: typer.Typer) -> None:
+    app.command(name="record", help="Execute real tools once and cache them for replay.")(record)

slmeval/cli/commands/report_cmd.py

@@ -0,0 +1,98 @@
+"""``slmeval report`` - re-render a stored run, with nothing re-executed.
+
+The acceptance criterion for this command is narrow and worth stating: the
+table it prints for a finished run has to be *identical* to the one that run
+printed when it finished. Nothing is recomputed from live objects, nothing is
+re-estimated, and nothing is re-asked of a model. If that ever stops being
+true, the trace store has stopped being the asset and the tool has become a
+thing you have to re-run to look at twice.
+
+It deliberately does not need the user's agent to import. A checkout whose
+agent has since been refactored still has valid results from last Tuesday, and
+refusing to print them would be the tool losing data it already holds.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.text import Text
+
+from slmeval.cli.loading import LATEST, open_project, resolve_run
+from slmeval.cli.render import console, symbols
+from slmeval.cli.wrap import handle_errors
+from slmeval.report.export import FORMATS, export
+from slmeval.report.model import build_scorecard
+from slmeval.report.scorecard import (
+    render_rollups,
+    render_scorecard,
+    render_summary_line,
+)
+
+
+@handle_errors
+def report(
+    run_id: Annotated[
+        str,
+        typer.Option("--run-id", help="A run id or prefix, or 'latest'."),
+    ] = LATEST,
+    fmt: Annotated[
+        str,
+        typer.Option("--format", "-f", help=f"One of: {', '.join(FORMATS)}."),
+    ] = "table",
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
+    ] = None,
+    nodes: Annotated[
+        bool, typer.Option("--nodes/--no-nodes", help="Show the per-node attribution table.")
+    ] = True,
+    cases: Annotated[
+        bool, typer.Option("--cases", help="Also show the per-case pass table.")
+    ] = False,
+    out_path: Annotated[
+        Path | None,
+        typer.Option("--out", "-o", help="Write to this file instead of stdout."),
+    ] = None,
+    ascii_mode: Annotated[
+        bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
+    ] = False,
+) -> None:
+    """Re-render the scorecard from a stored run."""
+    if fmt not in FORMATS:
+        raise typer.BadParameter(f"format must be one of: {', '.join(FORMATS)}")
+
+    out = console()
+    sym = symbols(ascii_mode or None)
+
+    with open_project(config, load_agent=False) as project:
+        resolved = resolve_run(project.store, run_id)
+        card = build_scorecard(project.store, resolved, prices_as_of=project.prices.as_of)
+
+        if fmt != "table":
+            text = export(card, fmt)
+            if out_path is not None:
+                out_path.write_text(text, encoding="utf-8")
+                out.print(Text(f"  {sym.ok} wrote {out_path}", style="dim"))
+                return
+            # print, not rich-print: a JSON or CSV payload is data on its way
+            # into a pipe, and Rich would wrap and style it into something no
+            # parser can read.
+            print(text)
+            return
+
+        out.print(render_scorecard(card, ascii_mode=ascii_mode or None, show_nodes=nodes))
+        if cases:
+            out.print()
+            out.print(render_rollups(card, ascii_mode=ascii_mode or None))
+        line = render_summary_line(card, ascii_mode=ascii_mode or None)
+        if str(line).strip():
+            out.print()
+            out.print(line)
+        out.print()
+
+
+def register(app: typer.Typer) -> None:
+    app.command(name="report", help="Re-render the scorecard from a stored run.")(report)