agentgauge-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentgauge/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """agentgauge: regression testing for LLM agents."""
2
+
3
+ __version__ = "0.1.0" # x-release-please-version
agentgauge/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from agentgauge.cli import app
2
+
3
+ app(prog_name="agentgauge")
@@ -0,0 +1,5 @@
1
+ """Command line interface."""
2
+
3
+ from agentgauge.cli.app import app
4
+
5
+ __all__ = ["app"]
agentgauge/cli/app.py ADDED
@@ -0,0 +1,171 @@
1
+ """The ``agentgauge`` command line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Annotated, NoReturn
7
+
8
+ import anyio
9
+ import typer
10
+ from pydantic import ValidationError
11
+ from rich.console import Console
12
+
13
+ from agentgauge import __version__
14
+ from agentgauge.cli.render import render_run, render_run_list
15
+ from agentgauge.core import ProviderConfig, RunResult, Suite, SuiteLoadError, load_suite
16
+ from agentgauge.providers import ProviderConfigError, build_provider
17
+ from agentgauge.runner import run_suite
18
+ from agentgauge.scorers import ScorerConfigError
19
+ from agentgauge.storage import RunStore, StorageError
20
+
21
+ DEFAULT_DB = Path(".agentgauge/runs.db")
22
+ CONFIG_ERROR = 2
23
+
24
+ app = typer.Typer(
25
+ name="agentgauge",
26
+ help="Regression testing for LLM agents.",
27
+ no_args_is_help=True,
28
+ add_completion=False,
29
+ )
30
+
31
+
32
+ def _print_version(value: bool) -> None:
33
+ if value:
34
+ typer.echo(f"agentgauge {__version__}")
35
+ raise typer.Exit()
36
+
37
+
38
+ runs_app = typer.Typer(help="Inspect stored runs.", no_args_is_help=True)
39
+ app.add_typer(runs_app, name="runs")
40
+
41
+ DbOption = Annotated[Path, typer.Option("--db", envvar="AGENTGAUGE_DB", help="Run database file.")]
42
+
43
+
44
+ @app.callback()
45
+ def main(
46
+ version: Annotated[
47
+ bool,
48
+ typer.Option(
49
+ "--version", callback=_print_version, is_eager=True, help="Show the version and exit."
50
+ ),
51
+ ] = False,
52
+ ) -> None:
53
+ """Regression testing for LLM agents."""
54
+
55
+
56
+ def _fail(message: str, code: int = CONFIG_ERROR) -> NoReturn:
57
+ typer.echo(f"error: {message}", err=True)
58
+ raise typer.Exit(code)
59
+
60
+
61
+ def _resolve_provider(
62
+ configured: ProviderConfig | None,
63
+ provider_type: str | None,
64
+ model: str | None,
65
+ base_url: str | None,
66
+ api_key_env: str | None,
67
+ ) -> ProviderConfig:
68
+ """Layer command-line options over the suite's provider block."""
69
+ options = {
70
+ "type": provider_type,
71
+ "model": model,
72
+ "base_url": base_url,
73
+ "api_key_env": api_key_env,
74
+ }
75
+ updates = {key: value for key, value in options.items() if value is not None}
76
+ merged = {**(configured.model_dump() if configured else {}), **updates}
77
+ if "type" not in merged:
78
+ if model is None:
79
+ _fail(
80
+ "no provider configured: add a 'provider' block to the suite, "
81
+ "or pass --model (and --provider fake for the fake provider)"
82
+ )
83
+ merged["type"] = "openai_compat"
84
+ try:
85
+ return ProviderConfig.model_validate(merged)
86
+ except ValidationError as exc:
87
+ _fail(f"invalid provider options: {exc.errors()[0]['msg']}")
88
+
89
+
90
+ async def _execute(suite: Suite, config: ProviderConfig, concurrency: int) -> RunResult:
91
+ provider = build_provider(config)
92
+ try:
93
+ return await run_suite(suite, provider, concurrency=concurrency)
94
+ finally:
95
+ close = getattr(provider, "aclose", None)
96
+ if close is not None:
97
+ await close()
98
+
99
+
100
+ @app.command()
101
+ def run(
102
+ suite_path: Annotated[
103
+ Path, typer.Argument(metavar="SUITE", help="Suite file (.yaml, .yml or .jsonl).")
104
+ ],
105
+ db: DbOption = DEFAULT_DB,
106
+ provider_type: Annotated[
107
+ str | None,
108
+ typer.Option("--provider", help="Provider type: openai_compat or fake."),
109
+ ] = None,
110
+ model: Annotated[str | None, typer.Option(help="Model name; overrides the suite.")] = None,
111
+ base_url: Annotated[
112
+ str | None, typer.Option(help="OpenAI-compatible endpoint; overrides the suite.")
113
+ ] = None,
114
+ api_key_env: Annotated[
115
+ str | None, typer.Option(help="Environment variable holding the API key.")
116
+ ] = None,
117
+ concurrency: Annotated[int, typer.Option(min=1, help="Maximum cases in flight at once.")] = 8,
118
+ ) -> None:
119
+ """Run SUITE against a model, score the outputs and store the run.
120
+
121
+ Exits 2 on configuration problems (bad suite, scorer or provider options,
122
+ unusable database). Cases that fail scoring or error do not change the exit
123
+ code; they are reported in the results table.
124
+ """
125
+ try:
126
+ suite = load_suite(suite_path)
127
+ config = _resolve_provider(suite.provider, provider_type, model, base_url, api_key_env)
128
+ store = RunStore(db)
129
+ except (SuiteLoadError, StorageError) as exc:
130
+ _fail(str(exc))
131
+ with store:
132
+ try:
133
+ result = anyio.run(_execute, suite, config, concurrency)
134
+ except (ScorerConfigError, ProviderConfigError) as exc:
135
+ _fail(str(exc))
136
+ store.save_run(result)
137
+ console = Console()
138
+ render_run(console, result)
139
+ console.print(f"Stored as run {result.run_id[:8]} in {db}")
140
+
141
+
142
+ @runs_app.command("list")
143
+ def runs_list(
144
+ db: DbOption = DEFAULT_DB,
145
+ limit: Annotated[int, typer.Option(min=1, help="Show at most this many runs.")] = 20,
146
+ ) -> None:
147
+ """List stored runs, newest first."""
148
+ try:
149
+ with RunStore(db) as store:
150
+ summaries = store.list_runs(limit=limit)
151
+ except StorageError as exc:
152
+ _fail(str(exc))
153
+ render_run_list(Console(), summaries)
154
+
155
+
156
+ @runs_app.command("show")
157
+ def runs_show(
158
+ run_id: Annotated[str, typer.Argument(metavar="ID", help="Run id or unique id prefix.")],
159
+ db: DbOption = DEFAULT_DB,
160
+ ) -> None:
161
+ """Show one stored run in detail, including each case's output."""
162
+ try:
163
+ store = RunStore(db)
164
+ except StorageError as exc:
165
+ _fail(str(exc))
166
+ with store:
167
+ try:
168
+ run = store.get_run(run_id)
169
+ except StorageError as exc: # unknown id or ambiguous prefix
170
+ _fail(str(exc), code=1)
171
+ render_run(Console(), run, verbose=True)
@@ -0,0 +1,140 @@
1
+ """Rich output for runs. Cell text is passed as ``Text`` so brackets in model output
2
+ or case ids are never interpreted as markup."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from collections.abc import Sequence
7
+ from dataclasses import dataclass
8
+ from datetime import UTC, datetime
9
+
10
+ from rich.console import Console
11
+ from rich.table import Table
12
+ from rich.text import Text
13
+
14
+ from agentgauge.core.models import CaseResult, RunResult
15
+ from agentgauge.storage import RunSummary
16
+
17
+ ID_WIDTH = 8
18
+ OUTPUT_PREVIEW = 60
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class RunStats:
23
+ total: int
24
+ passed: int
25
+ failed: int
26
+ errored: int
27
+ cost_usd: float | None
28
+ duration_s: float | None
29
+
30
+
31
+ def summarize(run: RunResult) -> RunStats:
32
+ """Count outcomes: ``errored`` cases raised, ``failed`` ones scored badly."""
33
+ errored = sum(1 for r in run.results if r.error is not None)
34
+ passed = sum(1 for r in run.results if r.passed)
35
+ costs = [r.cost_usd for r in run.results if r.cost_usd is not None]
36
+ duration = (run.finished_at - run.started_at).total_seconds() if run.finished_at else None
37
+ return RunStats(
38
+ total=len(run.results),
39
+ passed=passed,
40
+ failed=len(run.results) - passed - errored,
41
+ errored=errored,
42
+ cost_usd=sum(costs) if costs else None,
43
+ duration_s=duration,
44
+ )
45
+
46
+
47
+ def _moment(value: datetime) -> str:
48
+ return value.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
49
+
50
+
51
+ def _status(result: CaseResult) -> Text:
52
+ if result.error is not None:
53
+ return Text("ERROR", style="bold red")
54
+ return Text("PASS", style="green") if result.passed else Text("FAIL", style="red")
55
+
56
+
57
+ def _detail(result: CaseResult) -> str:
58
+ if result.error is not None:
59
+ return result.error
60
+ for score in result.scores:
61
+ if not score.passed:
62
+ return f"{score.scorer}: {score.reason}" if score.reason else score.scorer
63
+ return ""
64
+
65
+
66
+ def render_run(console: Console, run: RunResult, *, verbose: bool = False) -> None:
67
+ """Print a run header, a per-case table and a one-line summary."""
68
+ console.print(
69
+ Text.assemble(
70
+ ("Run ", "bold"),
71
+ (run.run_id[:ID_WIDTH], "cyan"),
72
+ f" suite {run.suite_name} model {run.model} started {_moment(run.started_at)}",
73
+ )
74
+ )
75
+ table = Table(show_lines=False)
76
+ table.add_column("Case")
77
+ table.add_column("Result")
78
+ table.add_column("Scores")
79
+ table.add_column("Latency", justify="right")
80
+ if verbose:
81
+ table.add_column("Output")
82
+ table.add_column("Detail")
83
+ for result in run.results:
84
+ scores = ", ".join(f"{s.scorer}={'pass' if s.passed else 'fail'}" for s in result.scores)
85
+ latency = f"{result.latency_ms:.0f} ms" if result.latency_ms is not None else "-"
86
+ cells: list[Text] = [
87
+ Text(result.case_id),
88
+ _status(result),
89
+ Text(scores or "-"),
90
+ Text(latency),
91
+ ]
92
+ if verbose:
93
+ cells.append(Text(_preview(result)))
94
+ cells.append(Text(_detail(result)))
95
+ table.add_row(*cells)
96
+ console.print(table)
97
+ console.print(_summary_line(summarize(run)))
98
+
99
+
100
+ def _preview(result: CaseResult) -> str:
101
+ if result.output is None:
102
+ return "-"
103
+ text = " ".join(result.output.text.split())
104
+ return text if len(text) <= OUTPUT_PREVIEW else text[: OUTPUT_PREVIEW - 1] + "…"
105
+
106
+
107
+ def _summary_line(stats: RunStats) -> Text:
108
+ line = Text(f"{stats.passed}/{stats.total} passed", style="bold")
109
+ line.append(f", {stats.failed} failed, {stats.errored} errors")
110
+ if stats.cost_usd is not None:
111
+ line.append(f", cost ${stats.cost_usd:.4f}")
112
+ if stats.duration_s is not None:
113
+ line.append(f", took {stats.duration_s:.2f}s")
114
+ return line
115
+
116
+
117
+ def render_run_list(console: Console, summaries: Sequence[RunSummary]) -> None:
118
+ """Print a table of stored runs, or a hint when there are none."""
119
+ if not summaries:
120
+ console.print("No runs stored yet. Run a suite with: agentgauge run SUITE")
121
+ return
122
+ table = Table()
123
+ table.add_column("ID", style="cyan")
124
+ table.add_column("Suite")
125
+ table.add_column("Model")
126
+ table.add_column("Started")
127
+ table.add_column("Cases", justify="right")
128
+ table.add_column("Passed", justify="right")
129
+ table.add_column("Errors", justify="right")
130
+ for s in summaries:
131
+ table.add_row(
132
+ Text(s.run_id[:ID_WIDTH]),
133
+ Text(s.suite_name),
134
+ Text(s.model),
135
+ Text(_moment(s.started_at)),
136
+ Text(str(s.total_cases)),
137
+ Text(str(s.passed_cases)),
138
+ Text(str(s.errored_cases)),
139
+ )
140
+ console.print(table)
@@ -0,0 +1,36 @@
1
+ """Core domain models."""
2
+
3
+ from agentgauge.core.loader import SuiteLoadError, load_suite
4
+ from agentgauge.core.models import (
5
+ CaseResult,
6
+ CompletionRequest,
7
+ GenerationParams,
8
+ Message,
9
+ ModelOutput,
10
+ ProviderConfig,
11
+ RunResult,
12
+ Score,
13
+ ScorerSpec,
14
+ Suite,
15
+ TestCase,
16
+ ToolCall,
17
+ ToolSpec,
18
+ )
19
+
20
+ __all__ = [
21
+ "CaseResult",
22
+ "CompletionRequest",
23
+ "GenerationParams",
24
+ "Message",
25
+ "ModelOutput",
26
+ "ProviderConfig",
27
+ "RunResult",
28
+ "Score",
29
+ "ScorerSpec",
30
+ "Suite",
31
+ "SuiteLoadError",
32
+ "TestCase",
33
+ "ToolCall",
34
+ "ToolSpec",
35
+ "load_suite",
36
+ ]
@@ -0,0 +1,109 @@
1
+ """Load suites from YAML or JSONL files with file/line error reporting."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any, cast
8
+
9
+ import yaml
10
+ from pydantic import ValidationError
11
+
12
+ from agentgauge.core.models import Suite, TestCase
13
+
14
+
15
+ class SuiteLoadError(Exception):
16
+ """Raised when a suite file cannot be read or is invalid."""
17
+
18
+ def __init__(self, path: Path, message: str, line: int | None = None) -> None:
19
+ self.path = path
20
+ self.line = line
21
+ self.message = message
22
+ location = f"{path}:{line}" if line is not None else str(path)
23
+ super().__init__(f"{location}: {message}")
24
+
25
+
26
+ def load_suite(path: str | Path) -> Suite:
27
+ """Load a suite from a ``.yaml``/``.yml`` or ``.jsonl`` file."""
28
+ p = Path(path)
29
+ suffix = p.suffix.lower()
30
+ if suffix not in {".yaml", ".yml", ".jsonl"}:
31
+ raise SuiteLoadError(p, f"unsupported file type {suffix!r}; use .yaml, .yml or .jsonl")
32
+ try:
33
+ text = p.read_text(encoding="utf-8")
34
+ except OSError as exc:
35
+ raise SuiteLoadError(p, f"cannot read file: {exc.strerror or exc}") from exc
36
+ if suffix == ".jsonl":
37
+ return _load_jsonl(p, text)
38
+ return _load_yaml(p, text)
39
+
40
+
41
+ def _load_jsonl(path: Path, text: str) -> Suite:
42
+ cases: list[TestCase] = []
43
+ for lineno, line in enumerate(text.splitlines(), start=1):
44
+ if not line.strip():
45
+ continue
46
+ try:
47
+ data = json.loads(line)
48
+ except json.JSONDecodeError as exc:
49
+ raise SuiteLoadError(path, f"invalid JSON: {exc.msg}", lineno) from exc
50
+ try:
51
+ cases.append(TestCase.model_validate(data))
52
+ except ValidationError as exc:
53
+ raise SuiteLoadError(path, _describe(exc), lineno) from exc
54
+ return Suite(name=path.stem, cases=tuple(cases))
55
+
56
+
57
+ def _load_yaml(path: Path, text: str) -> Suite:
58
+ try:
59
+ # types-PyYAML leaves compose()'s return type unannotated.
60
+ root = cast(
61
+ "yaml.Node | None",
62
+ yaml.compose(text, Loader=yaml.SafeLoader), # pyright: ignore[reportUnknownMemberType]
63
+ )
64
+ data: Any = yaml.safe_load(text)
65
+ except yaml.YAMLError as exc:
66
+ problem = getattr(exc, "problem", None) or "invalid YAML"
67
+ mark = getattr(exc, "problem_mark", None)
68
+ context = getattr(exc, "context_mark", None)
69
+ if context is not None and "<stream end>" in problem:
70
+ mark = context # unclosed construct: point at where it opened
71
+ line = mark.line + 1 if mark is not None else None
72
+ raise SuiteLoadError(path, f"invalid YAML: {problem}", line) from exc
73
+ if root is None or not isinstance(data, dict):
74
+ raise SuiteLoadError(path, "top level must be a mapping with 'name' and 'cases'", 1)
75
+ try:
76
+ return Suite.model_validate(data)
77
+ except ValidationError as exc:
78
+ first = exc.errors()[0]
79
+ line = _line_at(root, first["loc"])
80
+ raise SuiteLoadError(path, _describe(exc), line) from exc
81
+
82
+
83
+ def _line_at(root: yaml.Node, loc: tuple[int | str, ...]) -> int:
84
+ """Line of the deepest YAML node on ``loc``, falling back to the nearest parent."""
85
+ node = root
86
+ for part in loc:
87
+ child: yaml.Node | None = None
88
+ if isinstance(node, yaml.MappingNode):
89
+ for key, value in node.value:
90
+ if isinstance(key, yaml.ScalarNode) and key.value == part:
91
+ child = value
92
+ break
93
+ elif (
94
+ isinstance(node, yaml.SequenceNode)
95
+ and isinstance(part, int)
96
+ and 0 <= part < len(node.value)
97
+ ):
98
+ child = node.value[part]
99
+ if child is None:
100
+ break
101
+ node = child
102
+ return node.start_mark.line + 1
103
+
104
+
105
+ def _describe(exc: ValidationError) -> str:
106
+ first = exc.errors()[0]
107
+ field = ".".join(str(p) for p in first["loc"])
108
+ where = f"{field}: " if field else ""
109
+ return f"{where}{first['msg']}"
@@ -0,0 +1,220 @@
1
+ """Frozen domain models shared across the runner, scorers and storage."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Sequence
6
+ from datetime import datetime
7
+ from typing import TYPE_CHECKING, Any, Literal, cast
8
+
9
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
10
+
11
+
12
+ class _Model(BaseModel):
13
+ model_config = ConfigDict(frozen=True, extra="forbid")
14
+
15
+
16
+ class ToolCall(_Model):
17
+ """A tool call requested by the model. ``arguments`` is a JSON string."""
18
+
19
+ id: str
20
+ name: str
21
+ arguments: str
22
+
23
+
24
+ class Message(_Model):
25
+ """One chat message.
26
+
27
+ An assistant message may carry ``tool_calls``; a tool message must carry the
28
+ ``tool_call_id`` of the call it answers.
29
+ """
30
+
31
+ role: Literal["system", "user", "assistant", "tool"]
32
+ content: str
33
+ tool_call_id: str | None = None
34
+ tool_calls: tuple[ToolCall, ...] = ()
35
+
36
+ @model_validator(mode="after")
37
+ def _check_tool_fields(self) -> Message:
38
+ if self.tool_calls and self.role != "assistant":
39
+ raise ValueError("only assistant messages may carry tool_calls")
40
+ if self.role == "tool" and not self.tool_call_id:
41
+ raise ValueError("tool messages must have a tool_call_id")
42
+ return self
43
+
44
+
45
+ class ToolSpec(_Model):
46
+ """A tool the model may call, described with a JSON Schema for its arguments."""
47
+
48
+ name: str
49
+ description: str
50
+ parameters: dict[str, Any] = Field(default_factory=lambda: {})
51
+
52
+
53
+ class GenerationParams(_Model):
54
+ """Sampling parameters. ``None`` means "not set"."""
55
+
56
+ temperature: float | None = Field(default=None, ge=0)
57
+ max_tokens: int | None = Field(default=None, gt=0)
58
+ seed: int | None = None
59
+ stop: tuple[str, ...] | None = None
60
+
61
+ def merged_over(self, defaults: GenerationParams | None) -> GenerationParams:
62
+ """Return these params layered over ``defaults``, field by field.
63
+
64
+ A field set here wins; an unset (``None``) field falls back to ``defaults``.
65
+ """
66
+ if defaults is None:
67
+ return self
68
+ overrides = {k: v for k, v in self.model_dump().items() if v is not None}
69
+ return defaults.model_copy(update=overrides)
70
+
71
+
72
+ class CompletionRequest(_Model):
73
+ """Everything a provider needs to produce one completion."""
74
+
75
+ messages: tuple[Message, ...]
76
+ tools: tuple[ToolSpec, ...] = ()
77
+ params: GenerationParams | None = None
78
+
79
+
80
+ class TestCase(_Model):
81
+ """A single input to run against a model, with optional expectations.
82
+
83
+ ``input`` may be given as a plain string, which becomes a single user
84
+ message, or as a list of messages.
85
+ """
86
+
87
+ __test__ = False # not a pytest test class
88
+
89
+ id: str
90
+ input: tuple[Message, ...]
91
+ expected: str | None = None
92
+ metadata: dict[str, Any] = Field(default_factory=lambda: {})
93
+
94
+ if TYPE_CHECKING:
95
+
96
+ def __init__(
97
+ self,
98
+ *,
99
+ id: str,
100
+ input: str | Sequence[Message],
101
+ expected: str | None = None,
102
+ metadata: dict[str, Any] | None = None,
103
+ ) -> None: ...
104
+
105
+ @field_validator("input", mode="before")
106
+ @classmethod
107
+ def _wrap_string_input(cls, value: Any) -> Any:
108
+ if isinstance(value, str):
109
+ return ({"role": "user", "content": value},)
110
+ return value
111
+
112
+
113
+ class ScorerSpec(_Model):
114
+ """A scorer referenced by registry name, with optional constructor params."""
115
+
116
+ name: str
117
+ params: dict[str, Any] = Field(default_factory=lambda: {})
118
+
119
+
120
+ class ProviderConfig(_Model):
121
+ """Which model a suite runs against, so ``agentgauge run suite.yaml`` is self-contained.
122
+
123
+ ``type`` is ``openai_compat`` (any OpenAI-compatible endpoint) or ``fake``
124
+ (canned ``responses``, for examples and tests). ``api_key_env`` names the
125
+ environment variable holding the API key; keys are never stored in suites.
126
+ """
127
+
128
+ type: Literal["openai_compat", "fake"]
129
+ model: str | None = None
130
+ base_url: str | None = None
131
+ api_key_env: str | None = None
132
+ params: GenerationParams | None = None
133
+ responses: dict[str, str] = Field(default_factory=lambda: {})
134
+
135
+
136
+ class Suite(_Model):
137
+ """A named collection of test cases and the scorers applied to them.
138
+
139
+ Each entry in ``scorers`` may be a bare registry name or a mapping with
140
+ ``name`` and ``params``.
141
+ """
142
+
143
+ name: str
144
+ cases: tuple[TestCase, ...]
145
+ scorers: tuple[ScorerSpec, ...] = ()
146
+ description: str | None = None
147
+ provider: ProviderConfig | None = None
148
+
149
+ if TYPE_CHECKING:
150
+
151
+ def __init__(
152
+ self,
153
+ *,
154
+ name: str,
155
+ cases: Sequence[TestCase],
156
+ scorers: Sequence[str | ScorerSpec] = (),
157
+ description: str | None = None,
158
+ provider: ProviderConfig | None = None,
159
+ ) -> None: ...
160
+
161
+ @field_validator("scorers", mode="before")
162
+ @classmethod
163
+ def _expand_bare_names(cls, value: Any) -> Any:
164
+ if isinstance(value, list | tuple):
165
+ items = cast("Sequence[Any]", value)
166
+ return tuple({"name": v} if isinstance(v, str) else v for v in items)
167
+ return value
168
+
169
+
170
+ class ModelOutput(_Model):
171
+ """What a provider returned for one case."""
172
+
173
+ text: str
174
+ latency_ms: float = Field(ge=0)
175
+ prompt_tokens: int | None = Field(default=None, ge=0)
176
+ completion_tokens: int | None = Field(default=None, ge=0)
177
+ cost_usd: float | None = Field(default=None, ge=0)
178
+ tool_calls: tuple[ToolCall, ...] = ()
179
+ raw: dict[str, Any] | None = None
180
+
181
+
182
+ class Score(_Model):
183
+ """One scorer's verdict on one output."""
184
+
185
+ scorer: str
186
+ value: float
187
+ passed: bool
188
+ reason: str | None = None
189
+
190
+
191
+ class CaseResult(_Model):
192
+ """The outcome of running one case: output, scores and any error.
193
+
194
+ ``latency_ms`` is the wall-clock time the runner spent getting the output
195
+ (including provider retries), also recorded when the case failed;
196
+ ``cost_usd`` comes from the provider's output when it reports one.
197
+ """
198
+
199
+ case_id: str
200
+ output: ModelOutput | None = None
201
+ scores: tuple[Score, ...] = ()
202
+ error: str | None = None
203
+ latency_ms: float | None = Field(default=None, ge=0)
204
+ cost_usd: float | None = Field(default=None, ge=0)
205
+
206
+ @property
207
+ def passed(self) -> bool:
208
+ """True when the case ran without error and every score passed."""
209
+ return self.error is None and all(score.passed for score in self.scores)
210
+
211
+
212
+ class RunResult(_Model):
213
+ """The outcome of running a whole suite against one model."""
214
+
215
+ run_id: str
216
+ suite_name: str
217
+ model: str
218
+ started_at: datetime
219
+ finished_at: datetime | None = None
220
+ results: tuple[CaseResult, ...] = ()