evalbar 0.1.0__py3-none-any.whl

evalbar/html.py

@@ -0,0 +1,156 @@
+"""A single self-contained HTML page for a run: no external assets, no scripts.
+
+Everything that came from the pipeline, the dataset or a model is escaped, because LLM
+output is untrusted text.
+"""
+
+from __future__ import annotations
+
+import json
+from html import escape
+from typing import Any
+
+from .report import format_cost, summarize
+from .runner import CaseResult, RunResult
+
+_CSS = """
+:root { --bg:#f7f7f5; --card:#fff; --ink:#1d1d1f; --ink2:#5f6368; --line:#e3e3df;
+  --ok:#1a7f37; --ok-bg:#e6f4ea; --bad:#b42318; --bad-bg:#fdecea; --warn:#9a6700; --warn-bg:#fff4d6;
+  --bar:#2f6fde; --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+@media (prefers-color-scheme: dark) { :root { --bg:#141416; --card:#1d1d20; --ink:#ececf1; --ink2:#a1a1aa;
+  --line:#2e2e33; --ok:#4ac26b; --ok-bg:#12261a; --bad:#ff7b72; --bad-bg:#2d1414; --warn:#e3b341;
+  --warn-bg:#2b230f; --bar:#6c9ef8; } }
+* { box-sizing: border-box; }
+body { margin:0; background:var(--bg); color:var(--ink);
+  font: 15px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; }
+main { max-width: 1100px; margin: 0 auto; padding: 24px 16px 64px; }
+h1 { font-size: 22px; margin: 0 0 4px; } h2 { font-size: 16px; margin: 28px 0 10px; }
+.muted { color: var(--ink2); font-size: 13px; }
+.chips span { display:inline-block; margin:6px 6px 0 0; padding:2px 8px; border:1px solid var(--line);
+  border-radius:999px; font-size:12px; color:var(--ink2); }
+.cards { display:grid; grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); gap:10px; }
+.card { background:var(--card); border:1px solid var(--line); border-radius:10px; padding:12px; }
+.card strong { display:block; font-size:20px; } .card small { color:var(--ink2); }
+.tw { overflow-x:auto; background:var(--card); border:1px solid var(--line); border-radius:10px; }
+table { width:100%; border-collapse:collapse; font-size:14px; }
+tr:last-child > td { border-bottom:0; }
+th, td { text-align:left; padding:7px 10px; border-bottom:1px solid var(--line); vertical-align:top; }
+th { color:var(--ink2); font-weight:600; font-size:12px; text-transform:uppercase; letter-spacing:.03em; }
+td.num, th.num { text-align:right; font-variant-numeric: tabular-nums; }
+.bar { height:6px; background:var(--line); border-radius:3px; min-width:80px; }
+.bar i { display:block; height:6px; background:var(--bar); border-radius:3px; }
+details.case { background:var(--card); border:1px solid var(--line); border-radius:10px; margin:8px 0; }
+details.case > summary { cursor:pointer; padding:10px 12px; display:flex; gap:10px; align-items:baseline;
+  flex-wrap:wrap; }
+details.case > div { padding:0 12px 12px; }
+.badge { font-size:12px; font-weight:600; padding:1px 8px; border-radius:6px; }
+.pass .badge { color:var(--ok); background:var(--ok-bg); } .fail .badge { color:var(--bad); background:var(--bad-bg); }
+.error .badge { color:var(--warn); background:var(--warn-bg); }
+.cid { font-family:var(--mono); font-weight:600; }
+pre { font-family:var(--mono); font-size:12.5px; background:var(--bg); border:1px solid var(--line);
+  border-radius:8px; padding:8px 10px; white-space:pre-wrap; word-break:break-word; margin:4px 0 10px; }
+.grid2 { display:grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap:0 16px; }
+.label { font-size:12px; color:var(--ink2); text-transform:uppercase; letter-spacing:.03em; margin-top:8px; }
+.ok { color:var(--ok); } .bad { color:var(--bad); }
+"""
+
+
+def render_html(run: RunResult) -> str:
+    summary = summarize(run)
+    head = (f'<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'
+            f'<meta name="viewport" content="width=device-width, initial-scale=1">'
+            f"<title>evalbar: {_e(run.name)}</title><style>{_CSS}</style></head><body><main>")
+
+    chips = "".join(f"<span>{_e(key)}: {_e(value)}</span>"
+                    for key, value in {**run.metadata, **run.environment}.items())
+    header = (f"<h1>{_e(run.name)}</h1><div class=\"muted\">{summary.n_cases} cases, {summary.n_errors} errors, "
+              f"started {_e(run.started_at)}, finished {_e(run.finished_at)}</div><div class=\"chips\">{chips}</div>")
+
+    per_case = summary.n_cases or 1
+    cards = [
+        ("cost (list price)", format_cost(summary.cost_usd), f"{format_cost(summary.cost_usd / per_case)} per case"),
+        ("tokens", f"{summary.prompt_tokens + summary.completion_tokens:,}",
+         f"{summary.prompt_tokens:,} prompt / {summary.completion_tokens:,} completion"),
+        ("LLM latency p50", f"{summary.llm_latency_p50_s:.2f}s", f"p95 {summary.llm_latency_p95_s:.2f}s"),
+        ("errors", str(summary.n_errors), f"of {summary.n_cases} cases"),
+    ]
+    if summary.judge_calls:
+        cards.append(("judge cost", format_cost(summary.judge_cost_usd), f"{summary.judge_calls} calls, not in cost"))
+    if summary.n_unpriced_cases:
+        cards.append(("unpriced cases", str(summary.n_unpriced_cases), "cost excludes them"))
+    cards_html = "".join(f'<div class="card"><small>{_e(title)}</small><strong>{_e(value)}</strong>'
+                         f"<small>{_e(note)}</small></div>" for title, value, note in cards)
+
+    score_rows = "".join(
+        f"<tr><td>{_e(name)}</td><td class=\"num\">{s.n}</td><td class=\"num\">{s.passed}</td>"
+        f"<td class=\"num\">{s.pass_rate:.1%}</td><td><div class=\"bar\"><i style=\"width:{s.pass_rate * 100:.1f}%\">"
+        f"</i></div></td><td class=\"num\">{s.mean:.3f}</td></tr>"
+        for name, s in summary.scorers.items()
+    )
+    scores_html = ('<h2>Scores</h2><div class="tw"><table><tr><th>scorer</th><th class="num">cases</th>'
+                   '<th class="num">passed</th><th class="num">pass rate</th><th></th>'
+                   '<th class="num">mean</th></tr>'
+                   f"{score_rows}</table></div>")
+
+    ordered = sorted(run.cases, key=_case_rank)
+    cases_html = "<h2>Cases (failures first)</h2>" + "".join(_case(case) for case in ordered)
+    return head + header + f'<h2>Usage</h2><div class="cards">{cards_html}</div>' + scores_html + cases_html + \
+        "</main></body></html>\n"
+
+
+def _case_rank(case: CaseResult) -> int:
+    if case.error is not None:
+        return 0
+    return 1 if any(not score.passed for score in case.scores) else 2
+
+
+def _case(case: CaseResult) -> str:
+    status = ["error", "fail", "pass"][_case_rank(case)]
+    failed = [score.name for score in case.scores if not score.passed]
+    cost = case.trace.cost_usd
+    summary_line = (f'<span class="badge">{status}</span><span class="cid">{_e(case.case_id)}</span>'
+                    f'<span class="muted">{case.latency_s:.2f}s · '
+                    f'{"unpriced" if cost is None else format_cost(cost)}</span>'
+                    + (f'<span class="bad">{_e(", ".join(failed))}</span>' if failed else ""))
+
+    body = ['<div class="grid2">',
+            f'<div><div class="label">input</div><pre>{_e(_pretty(case.input))}</pre></div>',
+            f'<div><div class="label">expected</div><pre>{_e(_pretty(case.expected))}</pre></div></div>',
+            f'<div class="label">output</div><pre>{_e(_pretty(case.output))}</pre>']
+    if case.error:
+        body.append(f'<div class="label">error</div><pre class="bad">{_e(case.error)}</pre>')
+    if case.metadata:
+        body.append(f'<div class="label">metadata</div><pre>{_e(_pretty(case.metadata))}</pre>')
+    if case.scores:
+        rows = "".join(f'<tr><td>{_e(s.name)}</td><td class="{"ok" if s.passed else "bad"}">'
+                       f'{"pass" if s.passed else "fail"}</td><td class="num">{s.value:.2f}</td>'
+                       f"<td>{_e(s.reason)}</td></tr>" for s in case.scores)
+        body.append('<div class="label">scores</div><div class="tw"><table><tr><th>scorer</th><th>result</th>'
+                    f'<th class="num">value</th><th>reason</th></tr>{rows}</table></div>')
+    if case.trace.llm_calls:
+        rows = "".join(f"<tr><td>{_e(c.role)}</td><td>{_e(c.model)}</td><td class=\"num\">{c.prompt_tokens:,}</td>"
+                       f"<td class=\"num\">{c.completion_tokens:,}</td><td class=\"num\">{c.latency_s:.2f}s</td>"
+                       f"<td class=\"num\">{'unpriced' if c.cost_usd is None else format_cost(c.cost_usd)}</td></tr>"
+                       for c in case.trace.llm_calls)
+        body.append('<div class="label">LLM calls</div><div class="tw"><table><tr><th>role</th><th>model</th>'
+                    '<th class="num">prompt</th><th class="num">completion</th><th class="num">latency</th>'
+                    f'<th class="num">cost</th></tr>{rows}</table></div>')
+    if case.trace.tool_calls:
+        rows = "".join(f"<tr><td>{_e(t.name)}</td><td><pre>{_e(_pretty(t.args))}</pre></td>"
+                       f"<td><pre>{_e(t.error or _pretty(t.result))}</pre></td>"
+                       f"<td class=\"num\">{t.latency_s:.3f}s</td></tr>" for t in case.trace.tool_calls)
+        body.append('<div class="label">tool calls</div><div class="tw"><table><tr><th>tool</th><th>args</th>'
+                    f'<th>result</th><th class="num">latency</th></tr>{rows}</table></div>')
+
+    return (f'<details class="case {status}" id="case-{_e(case.case_id)}"><summary>{summary_line}</summary>'
+            f'<div>{"".join(body)}</div></details>')
+
+
+def _pretty(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False, indent=2, default=repr)
+
+
+def _e(value: Any) -> str:
+    return escape(str(value), quote=True)

evalbar/llm.py

@@ -0,0 +1,50 @@
+"""Traced LLM calls through LiteLLM."""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+# Use the price table bundled with the installed LiteLLM instead of fetching the latest one
+# at import time: runs stay reproducible, and the run file records the LiteLLM version.
+os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+
+import litellm  # noqa: E402
+
+from .trace import record_llm_call
+
+
+def completion(**kwargs: Any) -> Any:
+    """Call litellm.completion with the same arguments and record usage, latency and cost.
+
+    Cost comes from LiteLLM's price table (list price). A model with no known price is
+    recorded as None so reports never treat it as free.
+    """
+    started = time.perf_counter()
+    response = litellm.completion(**kwargs)
+    latency_s = time.perf_counter() - started
+
+    usage = getattr(response, "usage", None)
+    record_llm_call(
+        model=str(kwargs.get("model", getattr(response, "model", "unknown"))),
+        prompt_tokens=int(getattr(usage, "prompt_tokens", 0) or 0),
+        completion_tokens=int(getattr(usage, "completion_tokens", 0) or 0),
+        latency_s=latency_s,
+        cost_usd=_cost(response),
+    )
+    return response
+
+
+def _cost(response: Any) -> float | None:
+    # LiteLLM prices the call while it still knows the provider. completion_cost() on the
+    # response alone can guess wrong: Groq returns model="openai/gpt-oss-120b" with no
+    # provider hint, so the lookup goes to OpenAI and finds no price.
+    hidden = getattr(response, "_hidden_params", None) or {}
+    if hidden.get("response_cost") is not None:
+        return float(hidden["response_cost"])
+    try:
+        cost = litellm.completion_cost(completion_response=response)
+    except Exception:
+        return None
+    return float(cost)

evalbar/report.py

@@ -0,0 +1,174 @@
+"""Aggregate a run, render it for the terminal, and read/write JSON run files."""
+
+from __future__ import annotations
+
+import io
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from rich.console import Console, Group
+from rich.table import Table
+from rich.text import Text
+
+from .runner import RunResult
+
+
+@dataclass(frozen=True)
+class ScorerSummary:
+    n: int
+    passed: int
+    mean: float
+
+    @property
+    def pass_rate(self) -> float:
+        return self.passed / self.n if self.n else 0.0
+
+
+@dataclass(frozen=True)
+class Summary:
+    name: str
+    n_cases: int
+    n_errors: int
+    scorers: dict[str, ScorerSummary]
+    llm_calls: int
+    prompt_tokens: int
+    completion_tokens: int
+    cost_usd: float
+    n_unpriced_cases: int
+    judge_calls: int
+    judge_cost_usd: float
+    latency_p50_s: float
+    latency_p95_s: float
+    llm_latency_p50_s: float
+    llm_latency_p95_s: float
+
+
+def summarize(run: RunResult) -> Summary:
+    scorers = {}
+    for name in run.scorer_names:
+        scores = [score for case in run.cases for score in case.scores if score.name == name]
+        scorers[name] = ScorerSummary(
+            n=len(scores),
+            passed=sum(score.passed for score in scores),
+            mean=sum(score.value for score in scores) / len(scores) if scores else 0.0,
+        )
+    latencies = sorted(case.latency_s for case in run.cases)
+    # Time spent inside model calls only: excludes rate-limit waits and local work.
+    llm_latencies = sorted(case.trace.llm_latency_s for case in run.cases if case.trace.pipeline_calls)
+    return Summary(
+        name=run.name,
+        n_cases=len(run.cases),
+        n_errors=sum(case.error is not None for case in run.cases),
+        scorers=scorers,
+        llm_calls=sum(len(case.trace.pipeline_calls) for case in run.cases),
+        prompt_tokens=sum(case.trace.prompt_tokens for case in run.cases),
+        completion_tokens=sum(case.trace.completion_tokens for case in run.cases),
+        cost_usd=sum(call.cost_usd or 0.0 for case in run.cases for call in case.trace.pipeline_calls),
+        n_unpriced_cases=sum(
+            1 for case in run.cases if case.trace.pipeline_calls and case.trace.cost_usd is None
+        ),
+        judge_calls=sum(len(case.trace.judge_calls) for case in run.cases),
+        judge_cost_usd=sum(call.cost_usd or 0.0 for case in run.cases for call in case.trace.judge_calls),
+        latency_p50_s=percentile(latencies, 0.50),
+        latency_p95_s=percentile(latencies, 0.95),
+        llm_latency_p50_s=percentile(llm_latencies, 0.50),
+        llm_latency_p95_s=percentile(llm_latencies, 0.95),
+    )
+
+
+def report_renderable(run: RunResult, max_failures: int = 15) -> Group:
+    summary = summarize(run)
+    parts: list = []
+
+    header = Text()
+    header.append(f"evalbar run: {run.name}", style="bold")
+    header.append(f"  {summary.n_cases} cases, {summary.n_errors} errors, started {run.started_at}")
+    parts.append(header)
+    if run.metadata:
+        parts.append(Text("  " + ", ".join(f"{key}={value}" for key, value in run.metadata.items()), style="dim"))
+
+    scores = Table(title="Scores", title_justify="left")
+    for column in ("scorer", "cases", "passed", "pass rate", "mean"):
+        scores.add_column(column, justify="left" if column == "scorer" else "right", no_wrap=True)
+    for name, scorer in summary.scorers.items():
+        scores.add_row(name, str(scorer.n), str(scorer.passed), f"{scorer.pass_rate:.1%}", f"{scorer.mean:.3f}")
+    parts.append(scores)
+
+    usage = Table(title="Usage", title_justify="left")
+    usage.add_column("metric")
+    usage.add_column("value", justify="right")
+    per_case = summary.n_cases or 1
+    usage.add_row("LLM calls", str(summary.llm_calls))
+    usage.add_row("prompt tokens", f"{summary.prompt_tokens:,}")
+    usage.add_row("completion tokens", f"{summary.completion_tokens:,}")
+    usage.add_row("cost (list price)", format_cost(summary.cost_usd))
+    usage.add_row("cost per case", format_cost(summary.cost_usd / per_case))
+    if summary.judge_calls:
+        usage.add_row("judge calls", str(summary.judge_calls))
+        usage.add_row("judge cost (not in cost above)", format_cost(summary.judge_cost_usd))
+    usage.add_row("case latency p50 / p95", f"{summary.latency_p50_s:.2f}s / {summary.latency_p95_s:.2f}s")
+    usage.add_row("LLM latency p50 / p95", f"{summary.llm_latency_p50_s:.2f}s / {summary.llm_latency_p95_s:.2f}s")
+    parts.append(usage)
+    if summary.n_unpriced_cases:
+        parts.append(Text(f"{summary.n_unpriced_cases} case(s) had no known price; cost excludes them.",
+                          style="yellow"))
+
+    failures = [
+        (case.case_id, score.name, score.reason)
+        for case in run.cases
+        if case.error is None
+        for score in case.scores
+        if not score.passed
+    ]
+    errors = [(case.case_id, case.error) for case in run.cases if case.error is not None]
+    if failures:
+        table = Table(title=f"Failures ({len(failures)})", title_justify="left")
+        table.add_column("case")
+        table.add_column("scorer")
+        table.add_column("reason", overflow="fold")
+        for case_id, name, reason in failures[:max_failures]:
+            table.add_row(case_id, name, reason)
+        parts.append(table)
+        if len(failures) > max_failures:
+            parts.append(Text(f"... {len(failures) - max_failures} more in the run file", style="dim"))
+    if errors:
+        table = Table(title=f"Errors ({len(errors)})", title_justify="left")
+        table.add_column("case")
+        table.add_column("error", overflow="fold")
+        for case_id, error in errors[:max_failures]:
+            table.add_row(case_id, error or "")
+        parts.append(table)
+    return Group(*parts)
+
+
+def render(run: RunResult, width: int | None = None, max_failures: int = 15) -> str:
+    """The terminal report as plain text."""
+    console = Console(file=io.StringIO(), width=width or 120, record=True, color_system=None)
+    console.print(report_renderable(run, max_failures))
+    return console.export_text()
+
+
+def save_run(run: RunResult, path: str | Path) -> Path:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(run.to_dict(), indent=2, ensure_ascii=False, default=repr) + "\n",
+                    encoding="utf-8")
+    return path
+
+
+def load_run(path: str | Path) -> RunResult:
+    return RunResult.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
+
+
+def format_cost(usd: float) -> str:
+    return f"${usd:.4f}" if usd >= 0.01 or usd == 0 else f"${usd:.6f}"
+
+
+def percentile(sorted_values: list[float], fraction: float) -> float:
+    if not sorted_values:
+        return 0.0
+    position = (len(sorted_values) - 1) * fraction
+    lower = int(position)
+    upper = min(lower + 1, len(sorted_values) - 1)
+    return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * (position - lower)

evalbar/runner.py

@@ -0,0 +1,209 @@
+"""Run a pipeline over a dataset, one trace per case."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import platform
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field, replace
+from datetime import datetime, timezone
+from typing import Any
+
+from .dataset import Case
+from .scorers import Score, ScoreFn, Scorer, as_scorer
+from .trace import Trace
+
+SCHEMA_VERSION = 1
+
+
+@dataclass
+class Eval:
+    name: str
+    pipeline: Callable[[Any], Any]
+    dataset: Sequence[Case]
+    scorers: Sequence[Scorer | ScoreFn]
+    min_interval_s: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        self.scorers = [as_scorer(scorer) for scorer in self.scorers]
+        seen: set[str] = set()
+        for scorer in self.scorers:
+            if scorer.name in seen:
+                raise ValueError(f"duplicate scorer name '{scorer.name}'; pass name= to tell them apart")
+            seen.add(scorer.name)
+
+
+@dataclass
+class CaseResult:
+    case_id: str
+    input: Any
+    expected: dict[str, Any]
+    output: Any
+    error: str | None
+    latency_s: float
+    trace: Trace
+    scores: list[Score]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "case_id": self.case_id,
+            "input": self.input,
+            "expected": self.expected,
+            "metadata": self.metadata,
+            "output": self.output,
+            "error": self.error,
+            "latency_s": self.latency_s,
+            "trace": self.trace.to_dict(),
+            "scores": [score.__dict__ for score in self.scores],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> CaseResult:
+        return cls(
+            case_id=data["case_id"],
+            input=data["input"],
+            expected=data["expected"],
+            metadata=data.get("metadata", {}),
+            output=data["output"],
+            error=data["error"],
+            latency_s=data["latency_s"],
+            trace=Trace.from_dict(data["trace"]),
+            scores=[Score(**score) for score in data["scores"]],
+        )
+
+
+@dataclass
+class RunResult:
+    name: str
+    started_at: str
+    finished_at: str
+    scorer_names: list[str]
+    cases: list[CaseResult]
+    metadata: dict[str, Any] = field(default_factory=dict)
+    environment: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "name": self.name,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "scorer_names": self.scorer_names,
+            "metadata": self.metadata,
+            "environment": self.environment,
+            "cases": [case.to_dict() for case in self.cases],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunResult:
+        version = data.get("schema_version")
+        if version != SCHEMA_VERSION:
+            raise ValueError(f"unsupported run file schema_version {version!r} (expected {SCHEMA_VERSION})")
+        return cls(
+            name=data["name"],
+            started_at=data["started_at"],
+            finished_at=data["finished_at"],
+            scorer_names=list(data["scorer_names"]),
+            cases=[CaseResult.from_dict(case) for case in data["cases"]],
+            metadata=data.get("metadata", {}),
+            environment=data.get("environment", {}),
+        )
+
+
+def run(
+    evaluation: Eval,
+    *,
+    limit: int | None = None,
+    on_case: Callable[[CaseResult], None] | None = None,
+    clock: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+) -> RunResult:
+    """Run every case sequentially. Pipeline exceptions become case errors, never crashes."""
+    cases = list(evaluation.dataset)[:limit] if limit is not None else list(evaluation.dataset)
+    started_at = _now()
+    results: list[CaseResult] = []
+    last_start: float | None = None
+
+    for case in cases:
+        if last_start is not None and evaluation.min_interval_s > 0:
+            wait = evaluation.min_interval_s - (clock() - last_start)
+            if wait > 0:
+                sleep(wait)
+        last_start = clock()
+        result = _run_case(evaluation, case)
+        results.append(result)
+        if on_case is not None:
+            on_case(result)
+
+    return RunResult(
+        name=evaluation.name,
+        started_at=started_at,
+        finished_at=_now(),
+        scorer_names=[scorer.name for scorer in evaluation.scorers],
+        cases=results,
+        metadata=dict(evaluation.metadata),
+        environment=_environment(),
+    )
+
+
+def _run_case(evaluation: Eval, case: Case) -> CaseResult:
+    output: Any = None
+    error: str | None = None
+    started = time.perf_counter()
+    with Trace() as trace:
+        try:
+            output = evaluation.pipeline(case.input)
+        except Exception as exc:
+            # Spans recorded before the failure stay on the trace: a failed call still cost money.
+            error = f"{type(exc).__name__}: {exc}"
+    latency_s = time.perf_counter() - started
+
+    with trace.recording_as("judge"):
+        scores = _score(evaluation, case, output, error)
+
+    return CaseResult(
+        case_id=case.id,
+        input=case.input,
+        expected=case.expected,
+        output=output,
+        error=error,
+        latency_s=latency_s,
+        trace=trace,
+        scores=scores,
+        metadata=case.metadata,
+    )
+
+
+def _score(evaluation: Eval, case: Case, output: Any, error: str | None) -> list[Score]:
+    scores: list[Score] = []
+    for scorer in evaluation.scorers:
+        if error is not None:
+            if scorer.applies(case):
+                scores.append(Score(name=scorer.name, value=0.0, passed=False, reason="pipeline error"))
+            continue
+        try:
+            score = scorer(output, case)
+        except Exception as exc:
+            score = Score(name=scorer.name, value=0.0, passed=False,
+                          reason=f"scorer error: {type(exc).__name__}: {exc}")
+        if score is not None:
+            # Reports group by scorer name, so a custom scorer cannot rename its scores.
+            scores.append(replace(score, name=scorer.name))
+    return scores
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _environment() -> dict[str, str]:
+    env = {"python": platform.python_version()}
+    for package in ("evalbar", "litellm"):
+        try:
+            env[package] = importlib.metadata.version(package)
+        except importlib.metadata.PackageNotFoundError:
+            env[package] = "unknown"
+    return env