agentgauge-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentgauge/__init__.py +3 -0
- agentgauge/__main__.py +3 -0
- agentgauge/cli/__init__.py +5 -0
- agentgauge/cli/app.py +171 -0
- agentgauge/cli/render.py +140 -0
- agentgauge/core/__init__.py +36 -0
- agentgauge/core/loader.py +109 -0
- agentgauge/core/models.py +220 -0
- agentgauge/providers/__init__.py +16 -0
- agentgauge/providers/base.py +36 -0
- agentgauge/providers/factory.py +47 -0
- agentgauge/providers/fake.py +92 -0
- agentgauge/providers/openai_compat.py +253 -0
- agentgauge/py.typed +0 -0
- agentgauge/runner/__init__.py +5 -0
- agentgauge/runner/runner.py +103 -0
- agentgauge/scorers/__init__.py +31 -0
- agentgauge/scorers/base.py +27 -0
- agentgauge/scorers/deterministic.py +188 -0
- agentgauge/scorers/registry.py +53 -0
- agentgauge/scorers/schema.py +200 -0
- agentgauge/storage/__init__.py +6 -0
- agentgauge/storage/migrations.py +83 -0
- agentgauge/storage/store.py +183 -0
- agentgauge_eval-0.1.0.dist-info/METADATA +114 -0
- agentgauge_eval-0.1.0.dist-info/RECORD +29 -0
- agentgauge_eval-0.1.0.dist-info/WHEEL +4 -0
- agentgauge_eval-0.1.0.dist-info/entry_points.txt +2 -0
- agentgauge_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
agentgauge/__init__.py
ADDED
agentgauge/__main__.py
ADDED
agentgauge/cli/app.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""The ``agentgauge`` command line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Annotated, NoReturn
|
|
7
|
+
|
|
8
|
+
import anyio
|
|
9
|
+
import typer
|
|
10
|
+
from pydantic import ValidationError
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
|
|
13
|
+
from agentgauge import __version__
|
|
14
|
+
from agentgauge.cli.render import render_run, render_run_list
|
|
15
|
+
from agentgauge.core import ProviderConfig, RunResult, Suite, SuiteLoadError, load_suite
|
|
16
|
+
from agentgauge.providers import ProviderConfigError, build_provider
|
|
17
|
+
from agentgauge.runner import run_suite
|
|
18
|
+
from agentgauge.scorers import ScorerConfigError
|
|
19
|
+
from agentgauge.storage import RunStore, StorageError
|
|
20
|
+
|
|
21
|
+
DEFAULT_DB = Path(".agentgauge/runs.db")
|
|
22
|
+
CONFIG_ERROR = 2
|
|
23
|
+
|
|
24
|
+
app = typer.Typer(
|
|
25
|
+
name="agentgauge",
|
|
26
|
+
help="Regression testing for LLM agents.",
|
|
27
|
+
no_args_is_help=True,
|
|
28
|
+
add_completion=False,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _print_version(value: bool) -> None:
|
|
33
|
+
if value:
|
|
34
|
+
typer.echo(f"agentgauge {__version__}")
|
|
35
|
+
raise typer.Exit()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
runs_app = typer.Typer(help="Inspect stored runs.", no_args_is_help=True)
|
|
39
|
+
app.add_typer(runs_app, name="runs")
|
|
40
|
+
|
|
41
|
+
DbOption = Annotated[Path, typer.Option("--db", envvar="AGENTGAUGE_DB", help="Run database file.")]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@app.callback()
|
|
45
|
+
def main(
|
|
46
|
+
version: Annotated[
|
|
47
|
+
bool,
|
|
48
|
+
typer.Option(
|
|
49
|
+
"--version", callback=_print_version, is_eager=True, help="Show the version and exit."
|
|
50
|
+
),
|
|
51
|
+
] = False,
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Regression testing for LLM agents."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _fail(message: str, code: int = CONFIG_ERROR) -> NoReturn:
|
|
57
|
+
typer.echo(f"error: {message}", err=True)
|
|
58
|
+
raise typer.Exit(code)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _resolve_provider(
|
|
62
|
+
configured: ProviderConfig | None,
|
|
63
|
+
provider_type: str | None,
|
|
64
|
+
model: str | None,
|
|
65
|
+
base_url: str | None,
|
|
66
|
+
api_key_env: str | None,
|
|
67
|
+
) -> ProviderConfig:
|
|
68
|
+
"""Layer command-line options over the suite's provider block."""
|
|
69
|
+
options = {
|
|
70
|
+
"type": provider_type,
|
|
71
|
+
"model": model,
|
|
72
|
+
"base_url": base_url,
|
|
73
|
+
"api_key_env": api_key_env,
|
|
74
|
+
}
|
|
75
|
+
updates = {key: value for key, value in options.items() if value is not None}
|
|
76
|
+
merged = {**(configured.model_dump() if configured else {}), **updates}
|
|
77
|
+
if "type" not in merged:
|
|
78
|
+
if model is None:
|
|
79
|
+
_fail(
|
|
80
|
+
"no provider configured: add a 'provider' block to the suite, "
|
|
81
|
+
"or pass --model (and --provider fake for the fake provider)"
|
|
82
|
+
)
|
|
83
|
+
merged["type"] = "openai_compat"
|
|
84
|
+
try:
|
|
85
|
+
return ProviderConfig.model_validate(merged)
|
|
86
|
+
except ValidationError as exc:
|
|
87
|
+
_fail(f"invalid provider options: {exc.errors()[0]['msg']}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
async def _execute(suite: Suite, config: ProviderConfig, concurrency: int) -> RunResult:
|
|
91
|
+
provider = build_provider(config)
|
|
92
|
+
try:
|
|
93
|
+
return await run_suite(suite, provider, concurrency=concurrency)
|
|
94
|
+
finally:
|
|
95
|
+
close = getattr(provider, "aclose", None)
|
|
96
|
+
if close is not None:
|
|
97
|
+
await close()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@app.command()
|
|
101
|
+
def run(
|
|
102
|
+
suite_path: Annotated[
|
|
103
|
+
Path, typer.Argument(metavar="SUITE", help="Suite file (.yaml, .yml or .jsonl).")
|
|
104
|
+
],
|
|
105
|
+
db: DbOption = DEFAULT_DB,
|
|
106
|
+
provider_type: Annotated[
|
|
107
|
+
str | None,
|
|
108
|
+
typer.Option("--provider", help="Provider type: openai_compat or fake."),
|
|
109
|
+
] = None,
|
|
110
|
+
model: Annotated[str | None, typer.Option(help="Model name; overrides the suite.")] = None,
|
|
111
|
+
base_url: Annotated[
|
|
112
|
+
str | None, typer.Option(help="OpenAI-compatible endpoint; overrides the suite.")
|
|
113
|
+
] = None,
|
|
114
|
+
api_key_env: Annotated[
|
|
115
|
+
str | None, typer.Option(help="Environment variable holding the API key.")
|
|
116
|
+
] = None,
|
|
117
|
+
concurrency: Annotated[int, typer.Option(min=1, help="Maximum cases in flight at once.")] = 8,
|
|
118
|
+
) -> None:
|
|
119
|
+
"""Run SUITE against a model, score the outputs and store the run.
|
|
120
|
+
|
|
121
|
+
Exits 2 on configuration problems (bad suite, scorer or provider options,
|
|
122
|
+
unusable database). Cases that fail scoring or error do not change the exit
|
|
123
|
+
code; they are reported in the results table.
|
|
124
|
+
"""
|
|
125
|
+
try:
|
|
126
|
+
suite = load_suite(suite_path)
|
|
127
|
+
config = _resolve_provider(suite.provider, provider_type, model, base_url, api_key_env)
|
|
128
|
+
store = RunStore(db)
|
|
129
|
+
except (SuiteLoadError, StorageError) as exc:
|
|
130
|
+
_fail(str(exc))
|
|
131
|
+
with store:
|
|
132
|
+
try:
|
|
133
|
+
result = anyio.run(_execute, suite, config, concurrency)
|
|
134
|
+
except (ScorerConfigError, ProviderConfigError) as exc:
|
|
135
|
+
_fail(str(exc))
|
|
136
|
+
store.save_run(result)
|
|
137
|
+
console = Console()
|
|
138
|
+
render_run(console, result)
|
|
139
|
+
console.print(f"Stored as run {result.run_id[:8]} in {db}")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@runs_app.command("list")
|
|
143
|
+
def runs_list(
|
|
144
|
+
db: DbOption = DEFAULT_DB,
|
|
145
|
+
limit: Annotated[int, typer.Option(min=1, help="Show at most this many runs.")] = 20,
|
|
146
|
+
) -> None:
|
|
147
|
+
"""List stored runs, newest first."""
|
|
148
|
+
try:
|
|
149
|
+
with RunStore(db) as store:
|
|
150
|
+
summaries = store.list_runs(limit=limit)
|
|
151
|
+
except StorageError as exc:
|
|
152
|
+
_fail(str(exc))
|
|
153
|
+
render_run_list(Console(), summaries)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@runs_app.command("show")
|
|
157
|
+
def runs_show(
|
|
158
|
+
run_id: Annotated[str, typer.Argument(metavar="ID", help="Run id or unique id prefix.")],
|
|
159
|
+
db: DbOption = DEFAULT_DB,
|
|
160
|
+
) -> None:
|
|
161
|
+
"""Show one stored run in detail, including each case's output."""
|
|
162
|
+
try:
|
|
163
|
+
store = RunStore(db)
|
|
164
|
+
except StorageError as exc:
|
|
165
|
+
_fail(str(exc))
|
|
166
|
+
with store:
|
|
167
|
+
try:
|
|
168
|
+
run = store.get_run(run_id)
|
|
169
|
+
except StorageError as exc: # unknown id or ambiguous prefix
|
|
170
|
+
_fail(str(exc), code=1)
|
|
171
|
+
render_run(Console(), run, verbose=True)
|
agentgauge/cli/render.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Rich output for runs. Cell text is passed as ``Text`` so brackets in model output
|
|
2
|
+
or case ids are never interpreted as markup."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
from rich.text import Text
|
|
13
|
+
|
|
14
|
+
from agentgauge.core.models import CaseResult, RunResult
|
|
15
|
+
from agentgauge.storage import RunSummary
|
|
16
|
+
|
|
17
|
+
ID_WIDTH = 8
|
|
18
|
+
OUTPUT_PREVIEW = 60
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RunStats:
|
|
23
|
+
total: int
|
|
24
|
+
passed: int
|
|
25
|
+
failed: int
|
|
26
|
+
errored: int
|
|
27
|
+
cost_usd: float | None
|
|
28
|
+
duration_s: float | None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def summarize(run: RunResult) -> RunStats:
|
|
32
|
+
"""Count outcomes: ``errored`` cases raised, ``failed`` ones scored badly."""
|
|
33
|
+
errored = sum(1 for r in run.results if r.error is not None)
|
|
34
|
+
passed = sum(1 for r in run.results if r.passed)
|
|
35
|
+
costs = [r.cost_usd for r in run.results if r.cost_usd is not None]
|
|
36
|
+
duration = (run.finished_at - run.started_at).total_seconds() if run.finished_at else None
|
|
37
|
+
return RunStats(
|
|
38
|
+
total=len(run.results),
|
|
39
|
+
passed=passed,
|
|
40
|
+
failed=len(run.results) - passed - errored,
|
|
41
|
+
errored=errored,
|
|
42
|
+
cost_usd=sum(costs) if costs else None,
|
|
43
|
+
duration_s=duration,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _moment(value: datetime) -> str:
|
|
48
|
+
return value.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _status(result: CaseResult) -> Text:
|
|
52
|
+
if result.error is not None:
|
|
53
|
+
return Text("ERROR", style="bold red")
|
|
54
|
+
return Text("PASS", style="green") if result.passed else Text("FAIL", style="red")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _detail(result: CaseResult) -> str:
|
|
58
|
+
if result.error is not None:
|
|
59
|
+
return result.error
|
|
60
|
+
for score in result.scores:
|
|
61
|
+
if not score.passed:
|
|
62
|
+
return f"{score.scorer}: {score.reason}" if score.reason else score.scorer
|
|
63
|
+
return ""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def render_run(console: Console, run: RunResult, *, verbose: bool = False) -> None:
|
|
67
|
+
"""Print a run header, a per-case table and a one-line summary."""
|
|
68
|
+
console.print(
|
|
69
|
+
Text.assemble(
|
|
70
|
+
("Run ", "bold"),
|
|
71
|
+
(run.run_id[:ID_WIDTH], "cyan"),
|
|
72
|
+
f" suite {run.suite_name} model {run.model} started {_moment(run.started_at)}",
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
table = Table(show_lines=False)
|
|
76
|
+
table.add_column("Case")
|
|
77
|
+
table.add_column("Result")
|
|
78
|
+
table.add_column("Scores")
|
|
79
|
+
table.add_column("Latency", justify="right")
|
|
80
|
+
if verbose:
|
|
81
|
+
table.add_column("Output")
|
|
82
|
+
table.add_column("Detail")
|
|
83
|
+
for result in run.results:
|
|
84
|
+
scores = ", ".join(f"{s.scorer}={'pass' if s.passed else 'fail'}" for s in result.scores)
|
|
85
|
+
latency = f"{result.latency_ms:.0f} ms" if result.latency_ms is not None else "-"
|
|
86
|
+
cells: list[Text] = [
|
|
87
|
+
Text(result.case_id),
|
|
88
|
+
_status(result),
|
|
89
|
+
Text(scores or "-"),
|
|
90
|
+
Text(latency),
|
|
91
|
+
]
|
|
92
|
+
if verbose:
|
|
93
|
+
cells.append(Text(_preview(result)))
|
|
94
|
+
cells.append(Text(_detail(result)))
|
|
95
|
+
table.add_row(*cells)
|
|
96
|
+
console.print(table)
|
|
97
|
+
console.print(_summary_line(summarize(run)))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _preview(result: CaseResult) -> str:
|
|
101
|
+
if result.output is None:
|
|
102
|
+
return "-"
|
|
103
|
+
text = " ".join(result.output.text.split())
|
|
104
|
+
return text if len(text) <= OUTPUT_PREVIEW else text[: OUTPUT_PREVIEW - 1] + "…"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _summary_line(stats: RunStats) -> Text:
|
|
108
|
+
line = Text(f"{stats.passed}/{stats.total} passed", style="bold")
|
|
109
|
+
line.append(f", {stats.failed} failed, {stats.errored} errors")
|
|
110
|
+
if stats.cost_usd is not None:
|
|
111
|
+
line.append(f", cost ${stats.cost_usd:.4f}")
|
|
112
|
+
if stats.duration_s is not None:
|
|
113
|
+
line.append(f", took {stats.duration_s:.2f}s")
|
|
114
|
+
return line
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def render_run_list(console: Console, summaries: Sequence[RunSummary]) -> None:
|
|
118
|
+
"""Print a table of stored runs, or a hint when there are none."""
|
|
119
|
+
if not summaries:
|
|
120
|
+
console.print("No runs stored yet. Run a suite with: agentgauge run SUITE")
|
|
121
|
+
return
|
|
122
|
+
table = Table()
|
|
123
|
+
table.add_column("ID", style="cyan")
|
|
124
|
+
table.add_column("Suite")
|
|
125
|
+
table.add_column("Model")
|
|
126
|
+
table.add_column("Started")
|
|
127
|
+
table.add_column("Cases", justify="right")
|
|
128
|
+
table.add_column("Passed", justify="right")
|
|
129
|
+
table.add_column("Errors", justify="right")
|
|
130
|
+
for s in summaries:
|
|
131
|
+
table.add_row(
|
|
132
|
+
Text(s.run_id[:ID_WIDTH]),
|
|
133
|
+
Text(s.suite_name),
|
|
134
|
+
Text(s.model),
|
|
135
|
+
Text(_moment(s.started_at)),
|
|
136
|
+
Text(str(s.total_cases)),
|
|
137
|
+
Text(str(s.passed_cases)),
|
|
138
|
+
Text(str(s.errored_cases)),
|
|
139
|
+
)
|
|
140
|
+
console.print(table)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Core domain models."""
|
|
2
|
+
|
|
3
|
+
from agentgauge.core.loader import SuiteLoadError, load_suite
|
|
4
|
+
from agentgauge.core.models import (
|
|
5
|
+
CaseResult,
|
|
6
|
+
CompletionRequest,
|
|
7
|
+
GenerationParams,
|
|
8
|
+
Message,
|
|
9
|
+
ModelOutput,
|
|
10
|
+
ProviderConfig,
|
|
11
|
+
RunResult,
|
|
12
|
+
Score,
|
|
13
|
+
ScorerSpec,
|
|
14
|
+
Suite,
|
|
15
|
+
TestCase,
|
|
16
|
+
ToolCall,
|
|
17
|
+
ToolSpec,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"CaseResult",
|
|
22
|
+
"CompletionRequest",
|
|
23
|
+
"GenerationParams",
|
|
24
|
+
"Message",
|
|
25
|
+
"ModelOutput",
|
|
26
|
+
"ProviderConfig",
|
|
27
|
+
"RunResult",
|
|
28
|
+
"Score",
|
|
29
|
+
"ScorerSpec",
|
|
30
|
+
"Suite",
|
|
31
|
+
"SuiteLoadError",
|
|
32
|
+
"TestCase",
|
|
33
|
+
"ToolCall",
|
|
34
|
+
"ToolSpec",
|
|
35
|
+
"load_suite",
|
|
36
|
+
]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Load suites from YAML or JSONL files with file/line error reporting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
from pydantic import ValidationError
|
|
11
|
+
|
|
12
|
+
from agentgauge.core.models import Suite, TestCase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SuiteLoadError(Exception):
|
|
16
|
+
"""Raised when a suite file cannot be read or is invalid."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, path: Path, message: str, line: int | None = None) -> None:
|
|
19
|
+
self.path = path
|
|
20
|
+
self.line = line
|
|
21
|
+
self.message = message
|
|
22
|
+
location = f"{path}:{line}" if line is not None else str(path)
|
|
23
|
+
super().__init__(f"{location}: {message}")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load_suite(path: str | Path) -> Suite:
|
|
27
|
+
"""Load a suite from a ``.yaml``/``.yml`` or ``.jsonl`` file."""
|
|
28
|
+
p = Path(path)
|
|
29
|
+
suffix = p.suffix.lower()
|
|
30
|
+
if suffix not in {".yaml", ".yml", ".jsonl"}:
|
|
31
|
+
raise SuiteLoadError(p, f"unsupported file type {suffix!r}; use .yaml, .yml or .jsonl")
|
|
32
|
+
try:
|
|
33
|
+
text = p.read_text(encoding="utf-8")
|
|
34
|
+
except OSError as exc:
|
|
35
|
+
raise SuiteLoadError(p, f"cannot read file: {exc.strerror or exc}") from exc
|
|
36
|
+
if suffix == ".jsonl":
|
|
37
|
+
return _load_jsonl(p, text)
|
|
38
|
+
return _load_yaml(p, text)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _load_jsonl(path: Path, text: str) -> Suite:
|
|
42
|
+
cases: list[TestCase] = []
|
|
43
|
+
for lineno, line in enumerate(text.splitlines(), start=1):
|
|
44
|
+
if not line.strip():
|
|
45
|
+
continue
|
|
46
|
+
try:
|
|
47
|
+
data = json.loads(line)
|
|
48
|
+
except json.JSONDecodeError as exc:
|
|
49
|
+
raise SuiteLoadError(path, f"invalid JSON: {exc.msg}", lineno) from exc
|
|
50
|
+
try:
|
|
51
|
+
cases.append(TestCase.model_validate(data))
|
|
52
|
+
except ValidationError as exc:
|
|
53
|
+
raise SuiteLoadError(path, _describe(exc), lineno) from exc
|
|
54
|
+
return Suite(name=path.stem, cases=tuple(cases))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _load_yaml(path: Path, text: str) -> Suite:
|
|
58
|
+
try:
|
|
59
|
+
# types-PyYAML leaves compose()'s return type unannotated.
|
|
60
|
+
root = cast(
|
|
61
|
+
"yaml.Node | None",
|
|
62
|
+
yaml.compose(text, Loader=yaml.SafeLoader), # pyright: ignore[reportUnknownMemberType]
|
|
63
|
+
)
|
|
64
|
+
data: Any = yaml.safe_load(text)
|
|
65
|
+
except yaml.YAMLError as exc:
|
|
66
|
+
problem = getattr(exc, "problem", None) or "invalid YAML"
|
|
67
|
+
mark = getattr(exc, "problem_mark", None)
|
|
68
|
+
context = getattr(exc, "context_mark", None)
|
|
69
|
+
if context is not None and "<stream end>" in problem:
|
|
70
|
+
mark = context # unclosed construct: point at where it opened
|
|
71
|
+
line = mark.line + 1 if mark is not None else None
|
|
72
|
+
raise SuiteLoadError(path, f"invalid YAML: {problem}", line) from exc
|
|
73
|
+
if root is None or not isinstance(data, dict):
|
|
74
|
+
raise SuiteLoadError(path, "top level must be a mapping with 'name' and 'cases'", 1)
|
|
75
|
+
try:
|
|
76
|
+
return Suite.model_validate(data)
|
|
77
|
+
except ValidationError as exc:
|
|
78
|
+
first = exc.errors()[0]
|
|
79
|
+
line = _line_at(root, first["loc"])
|
|
80
|
+
raise SuiteLoadError(path, _describe(exc), line) from exc
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _line_at(root: yaml.Node, loc: tuple[int | str, ...]) -> int:
|
|
84
|
+
"""Line of the deepest YAML node on ``loc``, falling back to the nearest parent."""
|
|
85
|
+
node = root
|
|
86
|
+
for part in loc:
|
|
87
|
+
child: yaml.Node | None = None
|
|
88
|
+
if isinstance(node, yaml.MappingNode):
|
|
89
|
+
for key, value in node.value:
|
|
90
|
+
if isinstance(key, yaml.ScalarNode) and key.value == part:
|
|
91
|
+
child = value
|
|
92
|
+
break
|
|
93
|
+
elif (
|
|
94
|
+
isinstance(node, yaml.SequenceNode)
|
|
95
|
+
and isinstance(part, int)
|
|
96
|
+
and 0 <= part < len(node.value)
|
|
97
|
+
):
|
|
98
|
+
child = node.value[part]
|
|
99
|
+
if child is None:
|
|
100
|
+
break
|
|
101
|
+
node = child
|
|
102
|
+
return node.start_mark.line + 1
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _describe(exc: ValidationError) -> str:
|
|
106
|
+
first = exc.errors()[0]
|
|
107
|
+
field = ".".join(str(p) for p in first["loc"])
|
|
108
|
+
where = f"{field}: " if field else ""
|
|
109
|
+
return f"{where}{first['msg']}"
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Frozen domain models shared across the runner, scorers and storage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Literal, cast
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class _Model(BaseModel):
|
|
13
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ToolCall(_Model):
|
|
17
|
+
"""A tool call requested by the model. ``arguments`` is a JSON string."""
|
|
18
|
+
|
|
19
|
+
id: str
|
|
20
|
+
name: str
|
|
21
|
+
arguments: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Message(_Model):
|
|
25
|
+
"""One chat message.
|
|
26
|
+
|
|
27
|
+
An assistant message may carry ``tool_calls``; a tool message must carry the
|
|
28
|
+
``tool_call_id`` of the call it answers.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
role: Literal["system", "user", "assistant", "tool"]
|
|
32
|
+
content: str
|
|
33
|
+
tool_call_id: str | None = None
|
|
34
|
+
tool_calls: tuple[ToolCall, ...] = ()
|
|
35
|
+
|
|
36
|
+
@model_validator(mode="after")
|
|
37
|
+
def _check_tool_fields(self) -> Message:
|
|
38
|
+
if self.tool_calls and self.role != "assistant":
|
|
39
|
+
raise ValueError("only assistant messages may carry tool_calls")
|
|
40
|
+
if self.role == "tool" and not self.tool_call_id:
|
|
41
|
+
raise ValueError("tool messages must have a tool_call_id")
|
|
42
|
+
return self
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ToolSpec(_Model):
|
|
46
|
+
"""A tool the model may call, described with a JSON Schema for its arguments."""
|
|
47
|
+
|
|
48
|
+
name: str
|
|
49
|
+
description: str
|
|
50
|
+
parameters: dict[str, Any] = Field(default_factory=lambda: {})
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class GenerationParams(_Model):
|
|
54
|
+
"""Sampling parameters. ``None`` means "not set"."""
|
|
55
|
+
|
|
56
|
+
temperature: float | None = Field(default=None, ge=0)
|
|
57
|
+
max_tokens: int | None = Field(default=None, gt=0)
|
|
58
|
+
seed: int | None = None
|
|
59
|
+
stop: tuple[str, ...] | None = None
|
|
60
|
+
|
|
61
|
+
def merged_over(self, defaults: GenerationParams | None) -> GenerationParams:
|
|
62
|
+
"""Return these params layered over ``defaults``, field by field.
|
|
63
|
+
|
|
64
|
+
A field set here wins; an unset (``None``) field falls back to ``defaults``.
|
|
65
|
+
"""
|
|
66
|
+
if defaults is None:
|
|
67
|
+
return self
|
|
68
|
+
overrides = {k: v for k, v in self.model_dump().items() if v is not None}
|
|
69
|
+
return defaults.model_copy(update=overrides)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CompletionRequest(_Model):
|
|
73
|
+
"""Everything a provider needs to produce one completion."""
|
|
74
|
+
|
|
75
|
+
messages: tuple[Message, ...]
|
|
76
|
+
tools: tuple[ToolSpec, ...] = ()
|
|
77
|
+
params: GenerationParams | None = None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class TestCase(_Model):
|
|
81
|
+
"""A single input to run against a model, with optional expectations.
|
|
82
|
+
|
|
83
|
+
``input`` may be given as a plain string, which becomes a single user
|
|
84
|
+
message, or as a list of messages.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
__test__ = False # not a pytest test class
|
|
88
|
+
|
|
89
|
+
id: str
|
|
90
|
+
input: tuple[Message, ...]
|
|
91
|
+
expected: str | None = None
|
|
92
|
+
metadata: dict[str, Any] = Field(default_factory=lambda: {})
|
|
93
|
+
|
|
94
|
+
if TYPE_CHECKING:
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
*,
|
|
99
|
+
id: str,
|
|
100
|
+
input: str | Sequence[Message],
|
|
101
|
+
expected: str | None = None,
|
|
102
|
+
metadata: dict[str, Any] | None = None,
|
|
103
|
+
) -> None: ...
|
|
104
|
+
|
|
105
|
+
@field_validator("input", mode="before")
|
|
106
|
+
@classmethod
|
|
107
|
+
def _wrap_string_input(cls, value: Any) -> Any:
|
|
108
|
+
if isinstance(value, str):
|
|
109
|
+
return ({"role": "user", "content": value},)
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ScorerSpec(_Model):
|
|
114
|
+
"""A scorer referenced by registry name, with optional constructor params."""
|
|
115
|
+
|
|
116
|
+
name: str
|
|
117
|
+
params: dict[str, Any] = Field(default_factory=lambda: {})
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class ProviderConfig(_Model):
|
|
121
|
+
"""Which model a suite runs against, so ``agentgauge run suite.yaml`` is self-contained.
|
|
122
|
+
|
|
123
|
+
``type`` is ``openai_compat`` (any OpenAI-compatible endpoint) or ``fake``
|
|
124
|
+
(canned ``responses``, for examples and tests). ``api_key_env`` names the
|
|
125
|
+
environment variable holding the API key; keys are never stored in suites.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
type: Literal["openai_compat", "fake"]
|
|
129
|
+
model: str | None = None
|
|
130
|
+
base_url: str | None = None
|
|
131
|
+
api_key_env: str | None = None
|
|
132
|
+
params: GenerationParams | None = None
|
|
133
|
+
responses: dict[str, str] = Field(default_factory=lambda: {})
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class Suite(_Model):
|
|
137
|
+
"""A named collection of test cases and the scorers applied to them.
|
|
138
|
+
|
|
139
|
+
Each entry in ``scorers`` may be a bare registry name or a mapping with
|
|
140
|
+
``name`` and ``params``.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
name: str
|
|
144
|
+
cases: tuple[TestCase, ...]
|
|
145
|
+
scorers: tuple[ScorerSpec, ...] = ()
|
|
146
|
+
description: str | None = None
|
|
147
|
+
provider: ProviderConfig | None = None
|
|
148
|
+
|
|
149
|
+
if TYPE_CHECKING:
|
|
150
|
+
|
|
151
|
+
def __init__(
|
|
152
|
+
self,
|
|
153
|
+
*,
|
|
154
|
+
name: str,
|
|
155
|
+
cases: Sequence[TestCase],
|
|
156
|
+
scorers: Sequence[str | ScorerSpec] = (),
|
|
157
|
+
description: str | None = None,
|
|
158
|
+
provider: ProviderConfig | None = None,
|
|
159
|
+
) -> None: ...
|
|
160
|
+
|
|
161
|
+
@field_validator("scorers", mode="before")
|
|
162
|
+
@classmethod
|
|
163
|
+
def _expand_bare_names(cls, value: Any) -> Any:
|
|
164
|
+
if isinstance(value, list | tuple):
|
|
165
|
+
items = cast("Sequence[Any]", value)
|
|
166
|
+
return tuple({"name": v} if isinstance(v, str) else v for v in items)
|
|
167
|
+
return value
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class ModelOutput(_Model):
|
|
171
|
+
"""What a provider returned for one case."""
|
|
172
|
+
|
|
173
|
+
text: str
|
|
174
|
+
latency_ms: float = Field(ge=0)
|
|
175
|
+
prompt_tokens: int | None = Field(default=None, ge=0)
|
|
176
|
+
completion_tokens: int | None = Field(default=None, ge=0)
|
|
177
|
+
cost_usd: float | None = Field(default=None, ge=0)
|
|
178
|
+
tool_calls: tuple[ToolCall, ...] = ()
|
|
179
|
+
raw: dict[str, Any] | None = None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class Score(_Model):
|
|
183
|
+
"""One scorer's verdict on one output."""
|
|
184
|
+
|
|
185
|
+
scorer: str
|
|
186
|
+
value: float
|
|
187
|
+
passed: bool
|
|
188
|
+
reason: str | None = None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class CaseResult(_Model):
|
|
192
|
+
"""The outcome of running one case: output, scores and any error.
|
|
193
|
+
|
|
194
|
+
``latency_ms`` is the wall-clock time the runner spent getting the output
|
|
195
|
+
(including provider retries), also recorded when the case failed;
|
|
196
|
+
``cost_usd`` comes from the provider's output when it reports one.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
case_id: str
|
|
200
|
+
output: ModelOutput | None = None
|
|
201
|
+
scores: tuple[Score, ...] = ()
|
|
202
|
+
error: str | None = None
|
|
203
|
+
latency_ms: float | None = Field(default=None, ge=0)
|
|
204
|
+
cost_usd: float | None = Field(default=None, ge=0)
|
|
205
|
+
|
|
206
|
+
@property
|
|
207
|
+
def passed(self) -> bool:
|
|
208
|
+
"""True when the case ran without error and every score passed."""
|
|
209
|
+
return self.error is None and all(score.passed for score in self.scores)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class RunResult(_Model):
|
|
213
|
+
"""The outcome of running a whole suite against one model."""
|
|
214
|
+
|
|
215
|
+
run_id: str
|
|
216
|
+
suite_name: str
|
|
217
|
+
model: str
|
|
218
|
+
started_at: datetime
|
|
219
|
+
finished_at: datetime | None = None
|
|
220
|
+
results: tuple[CaseResult, ...] = ()
|