evalbar 0.1.0__py3-none-any.whl

evalbar/__init__.py

@@ -0,0 +1,34 @@
+"""evalbar: run an LLM pipeline over test cases, trace tokens/latency/cost, and score it."""
+
+from .dataset import Case, DatasetError, load_dataset
+from .diff import RunDiff, check_gates, diff_runs
+from .llm import completion
+from .report import load_run, render, save_run, summarize
+from .runner import CaseResult, Eval, RunResult, run
+from .scorers import Score, Scorer
+from .trace import Trace, current_trace, record_llm_call, record_tool_call, traced_tool
+
+__all__ = [
+    "Case",
+    "CaseResult",
+    "DatasetError",
+    "Eval",
+    "RunDiff",
+    "RunResult",
+    "Score",
+    "Scorer",
+    "Trace",
+    "check_gates",
+    "completion",
+    "current_trace",
+    "diff_runs",
+    "load_dataset",
+    "load_run",
+    "record_llm_call",
+    "record_tool_call",
+    "render",
+    "run",
+    "save_run",
+    "summarize",
+    "traced_tool",
+]

evalbar/cli.py

@@ -0,0 +1,181 @@
+"""Command line: `evalbar run`, `show`, `diff` and `html`."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from rich.console import Console
+
+from .diff import check_gates, diff_renderable, diff_runs, render_markdown
+from .html import render_html
+from .report import load_run, report_renderable, save_run
+from .runner import CaseResult, Eval, run
+
+
+class UsageError(Exception):
+    pass
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="evalbar", description="Run, trace and score LLM pipelines.")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    run_cmd = commands.add_parser("run", help="run an Eval defined in a Python file")
+    run_cmd.add_argument("target", help="path/to/eval.py or path/to/eval.py:eval_variable")
+    run_cmd.add_argument("--out", help="run file to write (default: runs/<name>-<timestamp>.json)")
+    run_cmd.add_argument("--limit", type=int, help="only run the first N cases (smoke test)")
+    run_cmd.add_argument("--quiet", action="store_true", help="no per-case progress lines")
+    run_cmd.add_argument("--env-file", default=".env",
+                         help="KEY=VALUE file loaded before the eval (default: .env if present); "
+                              "variables already set win")
+
+    show_cmd = commands.add_parser("show", help="print the report for a saved run file")
+    show_cmd.add_argument("run_file")
+
+    html_cmd = commands.add_parser("html", help="write a self-contained HTML report for a run file")
+    html_cmd.add_argument("run_file")
+    html_cmd.add_argument("--out", help="HTML file to write (default: the run file with .html)")
+
+    diff_cmd = commands.add_parser("diff", help="compare two run files; exit 1 when a gate fails")
+    diff_cmd.add_argument("base", help="baseline run file (e.g. committed on main)")
+    diff_cmd.add_argument("head", help="new run file")
+    diff_cmd.add_argument("--fail-on-drop", type=float, metavar="FRACTION",
+                          help="fail when a scorer's pass rate drops by more than this (0.02 = 2 points)")
+    diff_cmd.add_argument("--scorer", action="append", metavar="NAME",
+                          help="only gate on this scorer (repeatable; default: all shared scorers)")
+    diff_cmd.add_argument("--max-cost-increase", type=float, metavar="FRACTION",
+                          help="fail when cost per case rises by more than this (0.2 = 20%%)")
+    diff_cmd.add_argument("--format", choices=["text", "markdown"], default="text",
+                          help="markdown suits $GITHUB_STEP_SUMMARY and PR comments")
+
+    args = parser.parse_args(argv)
+    console = Console()
+    try:
+        if args.command == "run":
+            return _run(args, console)
+        if args.command == "diff":
+            return _diff(args, console)
+        if args.command == "html":
+            return _html(args, console)
+        return _show(args, console)
+    except UsageError as exc:
+        print(f"evalbar: {exc}", file=sys.stderr)
+        return 2
+
+
+def _run(args: argparse.Namespace, console: Console) -> int:
+    load_env_file(args.env_file)
+    evaluation = load_eval(args.target)
+    total = min(len(evaluation.dataset), args.limit) if args.limit is not None else len(evaluation.dataset)
+    progress = {"done": 0}
+
+    def on_case(result: CaseResult) -> None:
+        progress["done"] += 1
+        if args.quiet:
+            return
+        failed = [score.name for score in result.scores if not score.passed]
+        status = "ERROR" if result.error else ("fail: " + ", ".join(failed) if failed else "ok")
+        print(f"[{progress['done']}/{total}] {result.case_id} {result.latency_s:.2f}s {status}", file=sys.stderr)
+
+    result = run(evaluation, limit=args.limit, on_case=on_case)
+    out = args.out or f"runs/{evaluation.name}-{datetime.now():%Y%m%d-%H%M%S}.json"
+    path = save_run(result, out)
+    console.print(report_renderable(result))
+    console.print(f"run file: {path}", style="dim")
+    return 0
+
+
+def _show(args: argparse.Namespace, console: Console) -> int:
+    path = Path(args.run_file)
+    if not path.is_file():
+        raise UsageError(f"run file not found: {path}")
+    console.print(report_renderable(load_run(path)))
+    return 0
+
+
+def _html(args: argparse.Namespace, console: Console) -> int:
+    path = Path(args.run_file)
+    if not path.is_file():
+        raise UsageError(f"run file not found: {path}")
+    out = Path(args.out) if args.out else path.with_suffix(".html")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(render_html(load_run(path)), encoding="utf-8")
+    console.print(f"wrote {out}")
+    return 0
+
+
+def _diff(args: argparse.Namespace, console: Console) -> int:
+    runs = []
+    for run_file in (args.base, args.head):
+        path = Path(run_file)
+        if not path.is_file():
+            raise UsageError(f"run file not found: {path}")
+        runs.append(load_run(path))
+    diff = diff_runs(*runs)
+    try:
+        failures = check_gates(diff, fail_on_drop=args.fail_on_drop, scorers=args.scorer,
+                               max_cost_increase=args.max_cost_increase)
+    except ValueError as exc:
+        raise UsageError(str(exc)) from exc
+
+    if args.format == "markdown":
+        print(render_markdown(diff, failures), end="")
+    else:
+        console.print(diff_renderable(diff))
+    for failure in failures:
+        print(f"FAIL: {failure}", file=sys.stderr)
+    return 1 if failures else 0
+
+
+def load_env_file(path: str) -> None:
+    """Set variables from a KEY=VALUE file without overriding the environment. Values are never printed."""
+    env_path = Path(path)
+    if not env_path.is_file():
+        return
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.removeprefix("export ").partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        os.environ.setdefault(key.strip(), value)
+
+
+def load_eval(target: str) -> Eval:
+    """Import a Python file and return the Eval named after ':' (or its only Eval)."""
+    file_part, _, name = target.partition(":") if target.count(":") == 1 else (target, "", "")
+    path = Path(file_part)
+    if not path.is_file():
+        raise UsageError(f"eval file not found: {path}")
+
+    module_name = f"evalbar_user_{abs(hash(path.resolve()))}"
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        raise UsageError(f"cannot import {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.path.insert(0, str(path.resolve().parent))
+    try:
+        spec.loader.exec_module(module)
+    finally:
+        sys.path.pop(0)
+
+    evals = {key: value for key, value in vars(module).items() if isinstance(value, Eval)}
+    if name:
+        if name not in evals:
+            raise UsageError(f"{path} has no Eval named '{name}' (found: {', '.join(evals) or 'none'})")
+        return evals[name]
+    if len(evals) != 1:
+        found = ", ".join(evals) or "none"
+        raise UsageError(f"{path} defines {len(evals)} Evals ({found}); pick one with {path}:<name>")
+    return next(iter(evals.values()))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evalbar/dataset.py

@@ -0,0 +1,78 @@
+"""Load test cases from JSONL or YAML files."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+class DatasetError(ValueError):
+    """A dataset file is malformed. The message names the file and, for JSONL, the line."""
+
+
+@dataclass(frozen=True)
+class Case:
+    id: str
+    input: Any
+    expected: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def load_dataset(path: str | Path) -> list[Case]:
+    path = Path(path)
+    suffix = path.suffix.lower()
+    if suffix == ".jsonl":
+        records = _read_jsonl(path)
+    elif suffix in (".yaml", ".yml"):
+        records = _read_yaml(path)
+    else:
+        raise DatasetError(f"{path}: unsupported dataset format '{suffix}' (use .jsonl, .yaml or .yml)")
+
+    cases: list[Case] = []
+    seen: set[str] = set()
+    for where, record in records:
+        case = _to_case(path, where, record)
+        if case.id in seen:
+            raise DatasetError(f"{path} {where}: duplicate id '{case.id}'")
+        seen.add(case.id)
+        cases.append(case)
+    return cases
+
+
+def _read_jsonl(path: Path) -> list[tuple[str, Any]]:
+    records = []
+    for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            records.append((f"line {number}", json.loads(line)))
+        except json.JSONDecodeError as exc:
+            raise DatasetError(f"{path} line {number}: invalid JSON ({exc.msg})") from exc
+    return records
+
+
+def _read_yaml(path: Path) -> list[tuple[str, Any]]:
+    data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict):
+        data = data.get("cases")
+    if not isinstance(data, list):
+        raise DatasetError(f"{path}: expected a list of cases or a mapping with a 'cases' list")
+    return [(f"case {index}", record) for index, record in enumerate(data, start=1)]
+
+
+def _to_case(path: Path, where: str, record: Any) -> Case:
+    if not isinstance(record, dict):
+        raise DatasetError(f"{path} {where}: each case must be an object")
+    if "id" not in record:
+        raise DatasetError(f"{path} {where}: missing 'id'")
+    if "input" not in record:
+        raise DatasetError(f"{path} {where}: missing 'input'")
+    expected = record.get("expected") or {}
+    metadata = record.get("metadata") or {}
+    if not isinstance(expected, dict) or not isinstance(metadata, dict):
+        raise DatasetError(f"{path} {where}: 'expected' and 'metadata' must be objects")
+    return Case(id=str(record["id"]), input=record["input"], expected=expected, metadata=metadata)

evalbar/diff.py

@@ -0,0 +1,265 @@
+"""Compare two runs case by case and decide whether a CI gate passes."""
+
+from __future__ import annotations
+
+import io
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from rich import box
+from rich.console import Console, Group
+from rich.table import Table
+from rich.text import Text
+
+from .report import format_cost, percentile
+from .runner import CaseResult, RunResult
+
+_EPSILON = 1e-9
+_MAX_IDS = 10
+
+
+@dataclass(frozen=True)
+class ScorerDiff:
+    name: str
+    n: int
+    base_passed: int
+    head_passed: int
+    regressions: list[str]
+    fixes: list[str]
+
+    @property
+    def base_rate(self) -> float:
+        return self.base_passed / self.n if self.n else 0.0
+
+    @property
+    def head_rate(self) -> float:
+        return self.head_passed / self.n if self.n else 0.0
+
+    @property
+    def delta(self) -> float:
+        return self.head_rate - self.base_rate
+
+
+@dataclass(frozen=True)
+class RunDiff:
+    base_name: str
+    head_name: str
+    n_common: int
+    scorers: list[ScorerDiff]
+    added_cases: list[str]
+    removed_cases: list[str]
+    new_errors: list[str]
+    fixed_errors: list[str]
+    base_cost_per_case: float | None
+    head_cost_per_case: float | None
+    base_tokens_per_case: float
+    head_tokens_per_case: float
+    base_llm_latency_p50_s: float
+    head_llm_latency_p50_s: float
+
+    @property
+    def cost_change(self) -> float | None:
+        """Relative change in cost per case, or None when either side has unpriced calls."""
+        if self.base_cost_per_case is None or self.head_cost_per_case is None or not self.base_cost_per_case:
+            return None
+        return self.head_cost_per_case / self.base_cost_per_case - 1
+
+
+def diff_runs(base: RunResult, head: RunResult) -> RunDiff:
+    """Compare on the cases both runs share, so a changed dataset cannot fake a gain."""
+    base_cases = {case.case_id: case for case in base.cases}
+    head_cases = {case.case_id: case for case in head.cases}
+    common = [case_id for case_id in base_cases if case_id in head_cases]
+    pairs = [(base_cases[case_id], head_cases[case_id]) for case_id in common]
+
+    scorer_names = [name for name in base.scorer_names if name in head.scorer_names]
+    scorers = [_diff_scorer(name, pairs) for name in scorer_names]
+
+    return RunDiff(
+        base_name=base.name,
+        head_name=head.name,
+        n_common=len(common),
+        scorers=scorers,
+        added_cases=[case_id for case_id in head_cases if case_id not in base_cases],
+        removed_cases=[case_id for case_id in base_cases if case_id not in head_cases],
+        new_errors=[b.case_id for b, h in pairs if b.error is None and h.error is not None],
+        fixed_errors=[b.case_id for b, h in pairs if b.error is not None and h.error is None],
+        base_cost_per_case=_cost_per_case([b for b, _ in pairs]),
+        head_cost_per_case=_cost_per_case([h for _, h in pairs]),
+        base_tokens_per_case=_tokens_per_case([b for b, _ in pairs]),
+        head_tokens_per_case=_tokens_per_case([h for _, h in pairs]),
+        base_llm_latency_p50_s=_llm_p50([b for b, _ in pairs]),
+        head_llm_latency_p50_s=_llm_p50([h for _, h in pairs]),
+    )
+
+
+def check_gates(diff: RunDiff, *, fail_on_drop: float | None = None, scorers: Sequence[str] | None = None,
+                max_cost_increase: float | None = None) -> list[str]:
+    """Return one message per failed gate; an empty list means the gate passes."""
+    known = {scorer.name for scorer in diff.scorers}
+    for name in scorers or []:
+        if name not in known:
+            raise ValueError(f"unknown scorer '{name}' (in both runs: {', '.join(sorted(known)) or 'none'})")
+
+    failures = []
+    if fail_on_drop is not None:
+        for scorer in diff.scorers:
+            if scorers and scorer.name not in scorers:
+                continue
+            drop = scorer.base_rate - scorer.head_rate
+            if drop > fail_on_drop + _EPSILON:
+                cases = _ids(scorer.regressions)
+                failures.append(
+                    f"{scorer.name} dropped {drop * 100:.1f} pts ({scorer.base_rate:.1%} -> {scorer.head_rate:.1%}),"
+                    f" allowed {fail_on_drop * 100:.1f}; regressed: {cases}"
+                )
+    if max_cost_increase is not None:
+        failure = _cost_gate(diff, max_cost_increase)
+        if failure:
+            failures.append(failure)
+    return failures
+
+
+def _cost_gate(diff: RunDiff, max_increase: float) -> str | None:
+    base, head = diff.base_cost_per_case, diff.head_cost_per_case
+    if base is None or head is None:
+        # A requested gate must not pass on missing data.
+        return "cost gate requested but cost per case is unknown (some LLM calls have no known price)"
+    if base == 0:
+        return f"cost per case rose from $0 to {format_cost(head)}" if head > 0 else None
+    change = head / base - 1
+    if change > max_increase + _EPSILON:
+        return (f"cost per case rose {change:.0%} ({format_cost(base)} -> {format_cost(head)}), "
+                f"allowed {max_increase:.0%}")
+    return None
+
+
+def diff_renderable(diff: RunDiff) -> Group:
+    parts: list = [Text(f"evalbar diff: {diff.base_name} -> {diff.head_name}  ({diff.n_common} shared cases)",
+                        style="bold")]
+    if diff.added_cases or diff.removed_cases:
+        parts.append(Text(f"  only in head: {_ids(diff.added_cases)}; only in base: {_ids(diff.removed_cases)}",
+                          style="yellow"))
+
+    table = Table(title="Scores (pass rate)", title_justify="left", box=box.SIMPLE_HEAD)
+    table.add_column("scorer", no_wrap=True)
+    for column in ("n", "base", "head", "change", "down", "up"):
+        table.add_column(column, justify="right", no_wrap=True)
+    for scorer in diff.scorers:
+        style = "red" if scorer.delta < -_EPSILON else ("green" if scorer.delta > _EPSILON else "")
+        table.add_row(scorer.name, str(scorer.n), f"{scorer.base_rate:.1%}", f"{scorer.head_rate:.1%}",
+                      Text(f"{scorer.delta * 100:+.1f}", style=style), str(len(scorer.regressions)),
+                      str(len(scorer.fixes)))
+    parts.append(table)
+    for scorer in diff.scorers:
+        if scorer.regressions:
+            parts.append(Text(f"regressed {scorer.name}: {_ids(scorer.regressions)}", style="red"))
+        if scorer.fixes:
+            parts.append(Text(f"fixed {scorer.name}: {_ids(scorer.fixes)}", style="green"))
+
+    usage = Table(title="Usage per case", title_justify="left", box=box.SIMPLE_HEAD)
+    for column in ("metric", "base", "head", "change"):
+        usage.add_column(column, justify="left" if column == "metric" else "right")
+    for row in _usage_rows(diff):
+        usage.add_row(*row)
+    parts.append(usage)
+    if diff.new_errors:
+        parts.append(Text(f"new errors: {_ids(diff.new_errors)}", style="red"))
+    if diff.fixed_errors:
+        parts.append(Text(f"fixed errors: {_ids(diff.fixed_errors)}", style="green"))
+    return Group(*parts)
+
+
+def render_diff(diff: RunDiff, width: int | None = None) -> str:
+    console = Console(file=io.StringIO(), width=width or 120, record=True, color_system=None)
+    console.print(diff_renderable(diff))
+    return console.export_text()
+
+
+def render_markdown(diff: RunDiff, failures: Sequence[str] = ()) -> str:
+    """GitHub-flavoured markdown, e.g. for $GITHUB_STEP_SUMMARY or a PR comment."""
+    lines = [f"### evalbar diff: {diff.base_name} -> {diff.head_name}", "",
+             f"{diff.n_common} shared cases."]
+    if diff.added_cases or diff.removed_cases:
+        lines.append(f"Only in head: {_ids(diff.added_cases)}. Only in base: {_ids(diff.removed_cases)}.")
+    lines += ["", "| scorer | cases | base | head | change | regressed | fixed |", "|---|---|---|---|---|---|---|"]
+    for scorer in diff.scorers:
+        lines.append(f"| {scorer.name} | {scorer.n} | {scorer.base_rate:.1%} | {scorer.head_rate:.1%} | "
+                     f"{scorer.delta * 100:+.1f} pts | {_ids(scorer.regressions)} | {_ids(scorer.fixes)} |")
+    lines += ["", "| per case | base | head | change |", "|---|---|---|---|"]
+    lines += [f"| {' | '.join(row)} |" for row in _usage_rows(diff)]
+    if diff.new_errors:
+        lines += ["", f"New errors: {_ids(diff.new_errors)}"]
+    lines.append("")
+    if failures:
+        lines.append("**Gate failed**")
+        lines += [f"- {failure}" for failure in failures]
+    else:
+        lines.append("Gate passed.")
+    return "\n".join(lines) + "\n"
+
+
+def _diff_scorer(name: str, pairs: list[tuple[CaseResult, CaseResult]]) -> ScorerDiff:
+    n = base_passed = head_passed = 0
+    regressions, fixes = [], []
+    for base_case, head_case in pairs:
+        before = _passed(base_case, name)
+        after = _passed(head_case, name)
+        if before is None or after is None:
+            continue
+        n += 1
+        base_passed += before
+        head_passed += after
+        if before and not after:
+            regressions.append(base_case.case_id)
+        elif after and not before:
+            fixes.append(base_case.case_id)
+    return ScorerDiff(name, n, base_passed, head_passed, regressions, fixes)
+
+
+def _passed(case: CaseResult, name: str) -> bool | None:
+    return next((score.passed for score in case.scores if score.name == name), None)
+
+
+def _cost_per_case(cases: list[CaseResult]) -> float | None:
+    if not cases:
+        return None
+    costs = [case.trace.cost_usd for case in cases]
+    if any(cost is None for cost in costs):
+        return None
+    return sum(costs) / len(cases)  # type: ignore[arg-type]
+
+
+def _tokens_per_case(cases: list[CaseResult]) -> float:
+    if not cases:
+        return 0.0
+    return sum(case.trace.prompt_tokens + case.trace.completion_tokens for case in cases) / len(cases)
+
+
+def _llm_p50(cases: list[CaseResult]) -> float:
+    return percentile(sorted(case.trace.llm_latency_s for case in cases if case.trace.pipeline_calls), 0.5)
+
+
+def _usage_rows(diff: RunDiff) -> list[tuple[str, str, str, str]]:
+    def cost(value: float | None) -> str:
+        return "unknown" if value is None else format_cost(value)
+
+    def relative(before: float, after: float, floor: float = 0.0) -> str:
+        return f"{after / before - 1:+.0%}" if before > floor else ""
+
+    cost_change = "" if diff.cost_change is None else f"{diff.cost_change:+.0%}"
+    return [
+        ("cost (list price)", cost(diff.base_cost_per_case), cost(diff.head_cost_per_case), cost_change),
+        ("tokens", f"{diff.base_tokens_per_case:,.0f}", f"{diff.head_tokens_per_case:,.0f}",
+         relative(diff.base_tokens_per_case, diff.head_tokens_per_case)),
+        ("LLM latency p50", f"{diff.base_llm_latency_p50_s:.2f}s", f"{diff.head_llm_latency_p50_s:.2f}s",
+         # Sub-50 ms latencies are mocks or caches; a percentage of them is noise.
+         relative(diff.base_llm_latency_p50_s, diff.head_llm_latency_p50_s, floor=0.05)),
+    ]
+
+
+def _ids(case_ids: Sequence[str]) -> str:
+    if not case_ids:
+        return "-"
+    shown = ", ".join(case_ids[:_MAX_IDS])
+    return shown if len(case_ids) <= _MAX_IDS else f"{shown} (+{len(case_ids) - _MAX_IDS} more)"