agent-invariant-eval 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_eval/__init__.py +63 -0
- agent_eval/_version.py +1 -0
- agent_eval/benchmark/__init__.py +33 -0
- agent_eval/benchmark/frontier.py +431 -0
- agent_eval/cli.py +138 -0
- agent_eval/decorators.py +632 -0
- agent_eval/invariants/__init__.py +88 -0
- agent_eval/invariants/base.py +238 -0
- agent_eval/invariants/compliance.py +772 -0
- agent_eval/invariants/topology.py +390 -0
- agent_eval/py.typed +0 -0
- agent_eval/schemas.py +333 -0
- agent_eval/telemetry/__init__.py +10 -0
- agent_eval/telemetry/otel_exporter.py +320 -0
- agent_invariant_eval-0.3.0.dist-info/METADATA +593 -0
- agent_invariant_eval-0.3.0.dist-info/RECORD +19 -0
- agent_invariant_eval-0.3.0.dist-info/WHEEL +4 -0
- agent_invariant_eval-0.3.0.dist-info/entry_points.txt +2 -0
- agent_invariant_eval-0.3.0.dist-info/licenses/LICENSE +21 -0
agent_eval/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""agent-invariant-eval: deterministic, fail-closed evaluation for multi-agent workflows."""
|
|
2
|
+
|
|
3
|
+
from agent_eval._version import __version__
|
|
4
|
+
from agent_eval.decorators import (
|
|
5
|
+
EvaluationSession,
|
|
6
|
+
SimClock,
|
|
7
|
+
SpanScope,
|
|
8
|
+
current_session,
|
|
9
|
+
evaluate_agent,
|
|
10
|
+
)
|
|
11
|
+
from agent_eval.invariants import (
|
|
12
|
+
BaseInvariant,
|
|
13
|
+
InvariantRegistry,
|
|
14
|
+
InvariantViolation,
|
|
15
|
+
assert_no_context_bloat,
|
|
16
|
+
assert_no_pii,
|
|
17
|
+
assert_no_routing_loops,
|
|
18
|
+
assert_no_swallowed_tool_failures,
|
|
19
|
+
assert_numeric_grounding,
|
|
20
|
+
assert_pci_compliant,
|
|
21
|
+
assert_sla_bounds,
|
|
22
|
+
assert_valid_dag,
|
|
23
|
+
default_registry,
|
|
24
|
+
)
|
|
25
|
+
from agent_eval.schemas import (
|
|
26
|
+
AssertionResult,
|
|
27
|
+
EvalReport,
|
|
28
|
+
Severity,
|
|
29
|
+
Span,
|
|
30
|
+
SpanKind,
|
|
31
|
+
SpanStatus,
|
|
32
|
+
Stage,
|
|
33
|
+
Trajectory,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"AssertionResult",
|
|
38
|
+
"BaseInvariant",
|
|
39
|
+
"EvalReport",
|
|
40
|
+
"EvaluationSession",
|
|
41
|
+
"InvariantRegistry",
|
|
42
|
+
"InvariantViolation",
|
|
43
|
+
"Severity",
|
|
44
|
+
"SimClock",
|
|
45
|
+
"Span",
|
|
46
|
+
"SpanKind",
|
|
47
|
+
"SpanScope",
|
|
48
|
+
"SpanStatus",
|
|
49
|
+
"Stage",
|
|
50
|
+
"Trajectory",
|
|
51
|
+
"__version__",
|
|
52
|
+
"assert_no_context_bloat",
|
|
53
|
+
"assert_no_pii",
|
|
54
|
+
"assert_no_routing_loops",
|
|
55
|
+
"assert_no_swallowed_tool_failures",
|
|
56
|
+
"assert_numeric_grounding",
|
|
57
|
+
"assert_pci_compliant",
|
|
58
|
+
"assert_sla_bounds",
|
|
59
|
+
"assert_valid_dag",
|
|
60
|
+
"current_session",
|
|
61
|
+
"default_registry",
|
|
62
|
+
"evaluate_agent",
|
|
63
|
+
]
|
agent_eval/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Multi model replay and Pareto frontier."""
|
|
2
|
+
|
|
3
|
+
from agent_eval.benchmark.frontier import (
|
|
4
|
+
CaseResult,
|
|
5
|
+
EvalCase,
|
|
6
|
+
FrontierReport,
|
|
7
|
+
FrontierRunner,
|
|
8
|
+
ModelConfig,
|
|
9
|
+
ModelPrice,
|
|
10
|
+
ModelSummary,
|
|
11
|
+
dominates,
|
|
12
|
+
load_dataset,
|
|
13
|
+
pareto_frontier,
|
|
14
|
+
percentile,
|
|
15
|
+
recommend,
|
|
16
|
+
summarize,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"CaseResult",
|
|
21
|
+
"EvalCase",
|
|
22
|
+
"FrontierReport",
|
|
23
|
+
"FrontierRunner",
|
|
24
|
+
"ModelConfig",
|
|
25
|
+
"ModelPrice",
|
|
26
|
+
"ModelSummary",
|
|
27
|
+
"dominates",
|
|
28
|
+
"load_dataset",
|
|
29
|
+
"pareto_frontier",
|
|
30
|
+
"percentile",
|
|
31
|
+
"recommend",
|
|
32
|
+
"summarize",
|
|
33
|
+
]
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Cost to performance frontier.
|
|
2
|
+
|
|
3
|
+
Replays one JSONL dataset across several model configurations, scores every
|
|
4
|
+
run with the deterministic invariants, and reports pass rate, total inference
|
|
5
|
+
cost and p95 latency per model, with the Pareto optimal set and one
|
|
6
|
+
recommendation.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import inspect
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
16
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
21
|
+
|
|
22
|
+
from agent_eval.decorators import Clock, EvaluationSession, Invariants, _as_registry
|
|
23
|
+
from agent_eval.invariants.base import InvariantViolation
|
|
24
|
+
from agent_eval.schemas import Trajectory
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from agent_eval.telemetry.otel_exporter import TelemetryEmitter
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ModelPrice(BaseModel):
|
|
31
|
+
"""Token prices in USD per million tokens."""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(extra="forbid")
|
|
34
|
+
|
|
35
|
+
input_cost_per_mtok: float = Field(ge=0)
|
|
36
|
+
output_cost_per_mtok: float = Field(ge=0)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ModelConfig(BaseModel):
|
|
40
|
+
"""One configuration to benchmark. Prices are USD per million tokens.
|
|
41
|
+
|
|
42
|
+
``input_cost_per_mtok`` and ``output_cost_per_mtok`` are the default
|
|
43
|
+
prices. For a configuration that mixes models (a small model for triage,
|
|
44
|
+
a large one for resolution) add ``pricing``, keyed by the ``model`` set on
|
|
45
|
+
each span. Spans whose model is not in the map use the default prices.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
model_config = ConfigDict(extra="forbid")
|
|
49
|
+
|
|
50
|
+
name: str
|
|
51
|
+
input_cost_per_mtok: float = Field(ge=0)
|
|
52
|
+
output_cost_per_mtok: float = Field(ge=0)
|
|
53
|
+
pricing: dict[str, ModelPrice] = Field(default_factory=dict)
|
|
54
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
55
|
+
|
|
56
|
+
def cost_usd(
|
|
57
|
+
self, input_tokens: float, output_tokens: float, model: str | None = None
|
|
58
|
+
) -> float:
|
|
59
|
+
price = self.pricing.get(model) if model else None
|
|
60
|
+
input_price = price.input_cost_per_mtok if price else self.input_cost_per_mtok
|
|
61
|
+
output_price = price.output_cost_per_mtok if price else self.output_cost_per_mtok
|
|
62
|
+
return (input_tokens * input_price + output_tokens * output_price) / 1_000_000
|
|
63
|
+
|
|
64
|
+
def trajectory_cost_usd(self, trajectory: Trajectory) -> float:
|
|
65
|
+
"""Exact cost of a run, priced per span and without double counting."""
|
|
66
|
+
return sum(
|
|
67
|
+
self.cost_usd(input_tokens, output_tokens, span.model)
|
|
68
|
+
for span, input_tokens, output_tokens in trajectory.token_attribution()
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class EvalCase(BaseModel):
|
|
73
|
+
"""One dataset row.
|
|
74
|
+
|
|
75
|
+
``expect`` is the verdict a correct system produces for this case:
|
|
76
|
+
``"pass"`` for work the agent should complete cleanly, ``"block"`` for
|
|
77
|
+
adversarial input the engine must stop. For ``"block"`` cases,
|
|
78
|
+
``expected_violations`` names the invariants that must fire.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
model_config = ConfigDict(extra="forbid")
|
|
82
|
+
|
|
83
|
+
id: str
|
|
84
|
+
input: Any
|
|
85
|
+
expect: Literal["pass", "block"] = "pass"
|
|
86
|
+
expected_violations: list[str] = Field(default_factory=list)
|
|
87
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class CaseResult(BaseModel):
|
|
91
|
+
case_id: str
|
|
92
|
+
model: str
|
|
93
|
+
passed: bool
|
|
94
|
+
verdict: Literal["pass", "block"]
|
|
95
|
+
expected: Literal["pass", "block"]
|
|
96
|
+
violations: list[str] = Field(default_factory=list)
|
|
97
|
+
latency_ms: float = 0.0
|
|
98
|
+
input_tokens: int = 0
|
|
99
|
+
output_tokens: int = 0
|
|
100
|
+
cost_usd: float = 0.0
|
|
101
|
+
error: str | None = None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class ModelSummary(BaseModel):
|
|
105
|
+
model: str
|
|
106
|
+
cases: int
|
|
107
|
+
passed: int
|
|
108
|
+
pass_rate: float
|
|
109
|
+
total_cost_usd: float
|
|
110
|
+
p95_latency_ms: float
|
|
111
|
+
mean_latency_ms: float = 0.0
|
|
112
|
+
total_tokens: int = 0
|
|
113
|
+
pareto_optimal: bool = False
|
|
114
|
+
recommended: bool = False
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class FrontierReport(BaseModel):
|
|
118
|
+
min_pass_rate: float
|
|
119
|
+
summaries: list[ModelSummary]
|
|
120
|
+
results: list[CaseResult] = Field(default_factory=list)
|
|
121
|
+
recommended: str | None = None
|
|
122
|
+
|
|
123
|
+
def to_markdown(self) -> str:
|
|
124
|
+
"""Recommendation table, best pass rate first."""
|
|
125
|
+
header = (
|
|
126
|
+
"| Model | Pass rate | Passed | Total cost (USD) | p95 latency (ms) | Pareto optimal "
|
|
127
|
+
"| Recommendation |\n"
|
|
128
|
+
"|---|---:|---:|---:|---:|:---:|---|\n"
|
|
129
|
+
)
|
|
130
|
+
rows = []
|
|
131
|
+
ordered = sorted(self.summaries, key=lambda s: (-s.pass_rate, s.total_cost_usd, s.model))
|
|
132
|
+
for s in ordered:
|
|
133
|
+
if s.recommended:
|
|
134
|
+
note = "**recommended**"
|
|
135
|
+
elif not s.pareto_optimal:
|
|
136
|
+
note = "dominated"
|
|
137
|
+
elif s.pass_rate < self.min_pass_rate:
|
|
138
|
+
note = f"below {self.min_pass_rate:.0%} pass floor"
|
|
139
|
+
else:
|
|
140
|
+
note = "viable"
|
|
141
|
+
rows.append(
|
|
142
|
+
f"| {s.model} | {s.pass_rate:.1%} | {s.passed}/{s.cases} | "
|
|
143
|
+
f"{s.total_cost_usd:.6f} | {s.p95_latency_ms:.0f} | "
|
|
144
|
+
f"{'yes' if s.pareto_optimal else 'no'} | {note} |"
|
|
145
|
+
)
|
|
146
|
+
return header + "\n".join(rows)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# Sync or async: may return the output or an awaitable of it.
|
|
150
|
+
AgentRunner = Callable[[EvalCase, ModelConfig, EvaluationSession], Any]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def load_dataset(path: str | Path) -> list[EvalCase]:
|
|
154
|
+
"""Load a JSONL dataset. Blank lines and ``//`` comment lines are skipped."""
|
|
155
|
+
cases: list[EvalCase] = []
|
|
156
|
+
for number, line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), start=1):
|
|
157
|
+
stripped = line.strip()
|
|
158
|
+
if not stripped or stripped.startswith("//"):
|
|
159
|
+
continue
|
|
160
|
+
try:
|
|
161
|
+
cases.append(EvalCase.model_validate(json.loads(stripped)))
|
|
162
|
+
except Exception as exc:
|
|
163
|
+
raise ValueError(f"{path}:{number}: invalid case: {exc}") from exc
|
|
164
|
+
ids = [case.id for case in cases]
|
|
165
|
+
duplicates = sorted({i for i in ids if ids.count(i) > 1})
|
|
166
|
+
if duplicates:
|
|
167
|
+
raise ValueError(f"{path}: duplicate case ids: {', '.join(duplicates)}")
|
|
168
|
+
return cases
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def percentile(values: Sequence[float], pct: float) -> float:
|
|
172
|
+
"""Nearest rank percentile. ``percentile(xs, 95)`` is the p95."""
|
|
173
|
+
if not values:
|
|
174
|
+
return 0.0
|
|
175
|
+
if not 0 < pct <= 100:
|
|
176
|
+
raise ValueError("pct must be in (0, 100]")
|
|
177
|
+
ordered = sorted(values)
|
|
178
|
+
rank = math.ceil(pct / 100 * len(ordered))
|
|
179
|
+
return ordered[max(rank, 1) - 1]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def dominates(a: ModelSummary, b: ModelSummary) -> bool:
|
|
183
|
+
"""True when ``a`` is at least as good as ``b`` everywhere and better somewhere.
|
|
184
|
+
|
|
185
|
+
Objectives: maximise pass rate, minimise total cost, minimise p95 latency.
|
|
186
|
+
"""
|
|
187
|
+
no_worse = (
|
|
188
|
+
a.pass_rate >= b.pass_rate
|
|
189
|
+
and a.total_cost_usd <= b.total_cost_usd
|
|
190
|
+
and a.p95_latency_ms <= b.p95_latency_ms
|
|
191
|
+
)
|
|
192
|
+
better = (
|
|
193
|
+
a.pass_rate > b.pass_rate
|
|
194
|
+
or a.total_cost_usd < b.total_cost_usd
|
|
195
|
+
or a.p95_latency_ms < b.p95_latency_ms
|
|
196
|
+
)
|
|
197
|
+
return no_worse and better
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def pareto_frontier(summaries: Iterable[ModelSummary]) -> list[ModelSummary]:
|
|
201
|
+
"""The non dominated subset, in input order."""
|
|
202
|
+
items = list(summaries)
|
|
203
|
+
return [s for s in items if not any(dominates(other, s) for other in items if other is not s)]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def recommend(frontier: Sequence[ModelSummary], min_pass_rate: float) -> ModelSummary | None:
|
|
207
|
+
"""Cheapest frontier model that clears the pass floor.
|
|
208
|
+
|
|
209
|
+
If none clears it, fall back to the highest pass rate (then cheapest), so
|
|
210
|
+
the table always names the least bad option.
|
|
211
|
+
"""
|
|
212
|
+
if not frontier:
|
|
213
|
+
return None
|
|
214
|
+
viable = [s for s in frontier if s.pass_rate >= min_pass_rate]
|
|
215
|
+
if viable:
|
|
216
|
+
return min(
|
|
217
|
+
viable, key=lambda s: (s.total_cost_usd, -s.pass_rate, s.p95_latency_ms, s.model)
|
|
218
|
+
)
|
|
219
|
+
return min(frontier, key=lambda s: (-s.pass_rate, s.total_cost_usd, s.p95_latency_ms, s.model))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def summarize(
|
|
223
|
+
results: Iterable[CaseResult], min_pass_rate: float = 0.9
|
|
224
|
+
) -> FrontierReport:
|
|
225
|
+
"""Aggregate case results per model and mark the frontier and recommendation."""
|
|
226
|
+
all_results = list(results)
|
|
227
|
+
by_model: dict[str, list[CaseResult]] = {}
|
|
228
|
+
for result in all_results:
|
|
229
|
+
by_model.setdefault(result.model, []).append(result)
|
|
230
|
+
|
|
231
|
+
summaries: list[ModelSummary] = []
|
|
232
|
+
for model, rows in by_model.items():
|
|
233
|
+
latencies = [r.latency_ms for r in rows]
|
|
234
|
+
passed = sum(1 for r in rows if r.passed)
|
|
235
|
+
summaries.append(
|
|
236
|
+
ModelSummary(
|
|
237
|
+
model=model,
|
|
238
|
+
cases=len(rows),
|
|
239
|
+
passed=passed,
|
|
240
|
+
pass_rate=passed / len(rows),
|
|
241
|
+
total_cost_usd=round(sum(r.cost_usd for r in rows), 9),
|
|
242
|
+
p95_latency_ms=percentile(latencies, 95),
|
|
243
|
+
mean_latency_ms=round(sum(latencies) / len(latencies), 3),
|
|
244
|
+
total_tokens=sum(r.input_tokens + r.output_tokens for r in rows),
|
|
245
|
+
)
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
frontier = pareto_frontier(summaries)
|
|
249
|
+
for summary in frontier:
|
|
250
|
+
summary.pareto_optimal = True
|
|
251
|
+
pick = recommend(frontier, min_pass_rate)
|
|
252
|
+
if pick is not None:
|
|
253
|
+
pick.recommended = True
|
|
254
|
+
return FrontierReport(
|
|
255
|
+
min_pass_rate=min_pass_rate,
|
|
256
|
+
summaries=summaries,
|
|
257
|
+
results=all_results,
|
|
258
|
+
recommended=pick.model if pick else None,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class FrontierRunner:
|
|
263
|
+
"""Replays a dataset across model configurations.
|
|
264
|
+
|
|
265
|
+
``agent`` is any callable ``(case, model, session) -> output``, sync or
|
|
266
|
+
``async def``. It drives the real workflow with the given configuration
|
|
267
|
+
and records spans on the session. The runner sets the input, captures the
|
|
268
|
+
output, and scores.
|
|
269
|
+
|
|
270
|
+
A case passes when the verdict equals ``case.expect`` and every name in
|
|
271
|
+
``case.expected_violations`` actually fired. An agent exception is a
|
|
272
|
+
blocked run (fail closed).
|
|
273
|
+
|
|
274
|
+
``max_concurrency`` runs that many cases at once: worker threads in
|
|
275
|
+
:meth:`run`, tasks on the caller's event loop in :meth:`arun`. Results
|
|
276
|
+
keep dataset order either way. Use :meth:`arun` when the agent shares
|
|
277
|
+
loop bound clients (one ``httpx.AsyncClient``, one SDK client).
|
|
278
|
+
"""
|
|
279
|
+
|
|
280
|
+
def __init__(
|
|
281
|
+
self,
|
|
282
|
+
agent: AgentRunner,
|
|
283
|
+
*,
|
|
284
|
+
invariants: Invariants = None,
|
|
285
|
+
min_pass_rate: float = 0.9,
|
|
286
|
+
clock_factory: Callable[[], Clock] | None = None,
|
|
287
|
+
emitter: TelemetryEmitter | None = None,
|
|
288
|
+
max_concurrency: int = 1,
|
|
289
|
+
) -> None:
|
|
290
|
+
if not 0.0 <= min_pass_rate <= 1.0:
|
|
291
|
+
raise ValueError("min_pass_rate must be between 0 and 1")
|
|
292
|
+
if max_concurrency < 1:
|
|
293
|
+
raise ValueError("max_concurrency must be >= 1")
|
|
294
|
+
self.agent = agent
|
|
295
|
+
self.registry = _as_registry(invariants)
|
|
296
|
+
self.min_pass_rate = min_pass_rate
|
|
297
|
+
self.clock_factory = clock_factory
|
|
298
|
+
self.emitter = emitter
|
|
299
|
+
self.max_concurrency = max_concurrency
|
|
300
|
+
|
|
301
|
+
# ---- one case ---------------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
def _session(self, case: EvalCase, model: ModelConfig) -> EvaluationSession:
|
|
304
|
+
return EvaluationSession(
|
|
305
|
+
case.id,
|
|
306
|
+
invariants=self.registry,
|
|
307
|
+
enforce=True,
|
|
308
|
+
emitter=self.emitter,
|
|
309
|
+
clock=self.clock_factory() if self.clock_factory else None,
|
|
310
|
+
model=model.name,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
@staticmethod
|
|
314
|
+
def _score(
|
|
315
|
+
case: EvalCase, model: ModelConfig, session: EvaluationSession, error: str | None
|
|
316
|
+
) -> CaseResult:
|
|
317
|
+
report = session.report
|
|
318
|
+
if report is None: # defensive: no verdict means blocked
|
|
319
|
+
verdict: Literal["pass", "block"] = "block"
|
|
320
|
+
violations: list[str] = []
|
|
321
|
+
metrics: dict[str, float] = session.trajectory.metrics()
|
|
322
|
+
else:
|
|
323
|
+
verdict = "pass" if report.passed else "block"
|
|
324
|
+
violations = report.violated_names
|
|
325
|
+
metrics = report.metrics
|
|
326
|
+
matched = verdict == case.expect and set(case.expected_violations) <= set(violations)
|
|
327
|
+
return CaseResult(
|
|
328
|
+
case_id=case.id,
|
|
329
|
+
model=model.name,
|
|
330
|
+
passed=matched,
|
|
331
|
+
verdict=verdict,
|
|
332
|
+
expected=case.expect,
|
|
333
|
+
violations=violations,
|
|
334
|
+
latency_ms=float(metrics.get("latency_ms", 0.0)),
|
|
335
|
+
input_tokens=int(metrics.get("input_tokens", 0)),
|
|
336
|
+
output_tokens=int(metrics.get("output_tokens", 0)),
|
|
337
|
+
cost_usd=model.trajectory_cost_usd(session.trajectory),
|
|
338
|
+
error=error,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
def run_case(self, case: EvalCase, model: ModelConfig) -> CaseResult:
|
|
342
|
+
"""Run one case synchronously. An async agent gets its own event loop."""
|
|
343
|
+
try:
|
|
344
|
+
asyncio.get_running_loop()
|
|
345
|
+
in_event_loop = True
|
|
346
|
+
except RuntimeError:
|
|
347
|
+
in_event_loop = False
|
|
348
|
+
|
|
349
|
+
session = self._session(case, model)
|
|
350
|
+
error: str | None = None
|
|
351
|
+
misuse = False
|
|
352
|
+
try:
|
|
353
|
+
with session:
|
|
354
|
+
session.set_input(case.input)
|
|
355
|
+
output = self.agent(case, model, session)
|
|
356
|
+
if inspect.isawaitable(output):
|
|
357
|
+
if in_event_loop:
|
|
358
|
+
misuse = True
|
|
359
|
+
if inspect.iscoroutine(output):
|
|
360
|
+
output.close()
|
|
361
|
+
raise RuntimeError("async agent inside a running event loop")
|
|
362
|
+
output = asyncio.run(_await(output))
|
|
363
|
+
session.set_output(output)
|
|
364
|
+
except InvariantViolation:
|
|
365
|
+
pass
|
|
366
|
+
except Exception as exc: # noqa: BLE001 - recorded, the run counts as blocked
|
|
367
|
+
error = f"{type(exc).__name__}: {exc}"
|
|
368
|
+
if misuse:
|
|
369
|
+
raise RuntimeError(
|
|
370
|
+
"run() and run_case() cannot drive an async agent from inside a running "
|
|
371
|
+
"event loop. Use 'await runner.arun(...)' instead."
|
|
372
|
+
)
|
|
373
|
+
return self._score(case, model, session, error)
|
|
374
|
+
|
|
375
|
+
async def arun_case(self, case: EvalCase, model: ModelConfig) -> CaseResult:
|
|
376
|
+
"""Run one case on the current event loop. A sync agent blocks the loop."""
|
|
377
|
+
session = self._session(case, model)
|
|
378
|
+
error: str | None = None
|
|
379
|
+
try:
|
|
380
|
+
with session:
|
|
381
|
+
session.set_input(case.input)
|
|
382
|
+
output = self.agent(case, model, session)
|
|
383
|
+
if inspect.isawaitable(output):
|
|
384
|
+
output = await output
|
|
385
|
+
session.set_output(output)
|
|
386
|
+
except InvariantViolation:
|
|
387
|
+
pass
|
|
388
|
+
except Exception as exc: # noqa: BLE001 - recorded, the run counts as blocked
|
|
389
|
+
error = f"{type(exc).__name__}: {exc}"
|
|
390
|
+
return self._score(case, model, session, error)
|
|
391
|
+
|
|
392
|
+
# ---- whole dataset ----------------------------------------------------------
|
|
393
|
+
|
|
394
|
+
@staticmethod
|
|
395
|
+
def _plan(
|
|
396
|
+
cases: Sequence[EvalCase], models: Sequence[ModelConfig]
|
|
397
|
+
) -> list[tuple[EvalCase, ModelConfig]]:
|
|
398
|
+
if not cases:
|
|
399
|
+
raise ValueError("dataset is empty")
|
|
400
|
+
if not models:
|
|
401
|
+
raise ValueError("no model configurations given")
|
|
402
|
+
names = [m.name for m in models]
|
|
403
|
+
if len(set(names)) != len(names):
|
|
404
|
+
raise ValueError("model configuration names must be unique")
|
|
405
|
+
return [(case, model) for model in models for case in cases]
|
|
406
|
+
|
|
407
|
+
def run(self, cases: Sequence[EvalCase], models: Sequence[ModelConfig]) -> FrontierReport:
|
|
408
|
+
plan = self._plan(cases, models)
|
|
409
|
+
if self.max_concurrency == 1:
|
|
410
|
+
results = [self.run_case(case, model) for case, model in plan]
|
|
411
|
+
else:
|
|
412
|
+
with ThreadPoolExecutor(max_workers=self.max_concurrency) as pool:
|
|
413
|
+
results = list(pool.map(lambda pair: self.run_case(*pair), plan))
|
|
414
|
+
return summarize(results, self.min_pass_rate)
|
|
415
|
+
|
|
416
|
+
async def arun(
|
|
417
|
+
self, cases: Sequence[EvalCase], models: Sequence[ModelConfig]
|
|
418
|
+
) -> FrontierReport:
|
|
419
|
+
plan = self._plan(cases, models)
|
|
420
|
+
gate = asyncio.Semaphore(self.max_concurrency)
|
|
421
|
+
|
|
422
|
+
async def guarded(case: EvalCase, model: ModelConfig) -> CaseResult:
|
|
423
|
+
async with gate:
|
|
424
|
+
return await self.arun_case(case, model)
|
|
425
|
+
|
|
426
|
+
results = await asyncio.gather(*(guarded(case, model) for case, model in plan))
|
|
427
|
+
return summarize(list(results), self.min_pass_rate)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
async def _await(awaitable: Any) -> Any:
|
|
431
|
+
return await awaitable
|
agent_eval/cli.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Command line interface: ``agent-eval scan`` and ``agent-eval benchmark``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Any
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
import typer
|
|
13
|
+
except ImportError as exc:
|
|
14
|
+
raise ImportError(
|
|
15
|
+
"The CLI requires the 'cli' extra: pip install 'agent-invariant-eval[cli]'"
|
|
16
|
+
) from exc
|
|
17
|
+
|
|
18
|
+
from agent_eval.benchmark.frontier import FrontierRunner, ModelConfig, load_dataset
|
|
19
|
+
from agent_eval.decorators import SimClock
|
|
20
|
+
from agent_eval.invariants import default_registry, find_pci, find_pii
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(
|
|
23
|
+
add_completion=False,
|
|
24
|
+
no_args_is_help=True,
|
|
25
|
+
help="Deterministic, fail-closed evaluation for multi-agent workflows.",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_object(reference: str) -> Any:
|
|
30
|
+
"""Import ``package.module:attribute``. The current directory is importable."""
|
|
31
|
+
module_name, _, attribute = reference.partition(":")
|
|
32
|
+
if not module_name or not attribute:
|
|
33
|
+
raise typer.BadParameter(f"expected 'module:attribute', got {reference!r}")
|
|
34
|
+
if str(Path.cwd()) not in sys.path:
|
|
35
|
+
sys.path.insert(0, str(Path.cwd()))
|
|
36
|
+
try:
|
|
37
|
+
return getattr(importlib.import_module(module_name), attribute)
|
|
38
|
+
except (ImportError, AttributeError) as exc:
|
|
39
|
+
raise typer.BadParameter(f"cannot load {reference!r}: {exc}") from exc
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _load_models(reference: str) -> list[ModelConfig]:
|
|
43
|
+
"""Model configurations from a JSON file or a ``module:attribute`` list."""
|
|
44
|
+
path = Path(reference)
|
|
45
|
+
if path.is_file():
|
|
46
|
+
try:
|
|
47
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
48
|
+
except json.JSONDecodeError as exc:
|
|
49
|
+
raise typer.BadParameter(f"{reference}: not valid JSON: {exc}") from exc
|
|
50
|
+
elif ":" in reference:
|
|
51
|
+
raw = _load_object(reference)
|
|
52
|
+
else:
|
|
53
|
+
raise typer.BadParameter(
|
|
54
|
+
f"{reference!r} is neither an existing JSON file nor a 'module:attribute' reference"
|
|
55
|
+
)
|
|
56
|
+
if not isinstance(raw, (list, tuple)) or not raw:
|
|
57
|
+
raise typer.BadParameter(f"{reference}: expected a non empty list of model configurations")
|
|
58
|
+
try:
|
|
59
|
+
return [ModelConfig.model_validate(item) for item in raw]
|
|
60
|
+
except ValueError as exc:
|
|
61
|
+
raise typer.BadParameter(f"{reference}: invalid model configuration: {exc}") from exc
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command()
|
|
65
|
+
def scan(
|
|
66
|
+
text: Annotated[str | None, typer.Argument(help="Text to scan. Omit to use --file.")] = None,
|
|
67
|
+
file: Annotated[Path | None, typer.Option("--file", "-f", help="File to scan.")] = None,
|
|
68
|
+
allowed_domain: Annotated[
|
|
69
|
+
list[str] | None,
|
|
70
|
+
typer.Option("--allowed-domain", help="Email domain to exempt (repeatable)."),
|
|
71
|
+
] = None,
|
|
72
|
+
require_known_bin: Annotated[
|
|
73
|
+
bool,
|
|
74
|
+
typer.Option("--require-known-bin", help="Only flag PANs with a known card network BIN."),
|
|
75
|
+
] = False,
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Scan text for PII and PCI data. Findings are printed masked. Exit 1 on any finding."""
|
|
78
|
+
if (text is None) == (file is None):
|
|
79
|
+
raise typer.BadParameter("give either TEXT or --file, not both")
|
|
80
|
+
content = text if text is not None else file.read_text(encoding="utf-8") # type: ignore[union-attr]
|
|
81
|
+
findings = [
|
|
82
|
+
*find_pii(content, allowed_domains=allowed_domain or ()),
|
|
83
|
+
*find_pci(content, require_known_bin=require_known_bin),
|
|
84
|
+
]
|
|
85
|
+
if not findings:
|
|
86
|
+
typer.echo("clean: no PII or PCI findings")
|
|
87
|
+
return
|
|
88
|
+
for finding in sorted(findings, key=lambda f: f.start):
|
|
89
|
+
typer.echo(f"{finding.kind:<20} offset {finding.start:<6} {finding.masked}")
|
|
90
|
+
raise typer.Exit(code=1)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@app.command()
|
|
94
|
+
def benchmark(
|
|
95
|
+
dataset: Annotated[Path, typer.Argument(help="JSONL evaluation dataset.")],
|
|
96
|
+
agent: Annotated[
|
|
97
|
+
str, typer.Option(help="Agent runner as 'module:callable' taking (case, model, session).")
|
|
98
|
+
],
|
|
99
|
+
models: Annotated[
|
|
100
|
+
str,
|
|
101
|
+
typer.Option(
|
|
102
|
+
help="Model configurations: a JSON file, or 'module:attribute' naming a list."
|
|
103
|
+
),
|
|
104
|
+
],
|
|
105
|
+
registry: Annotated[
|
|
106
|
+
str | None,
|
|
107
|
+
typer.Option(help="Registry factory as 'module:callable'. Defaults to the built in set."),
|
|
108
|
+
] = None,
|
|
109
|
+
min_pass_rate: Annotated[float, typer.Option(min=0.0, max=1.0)] = 0.9,
|
|
110
|
+
sim_clock: Annotated[
|
|
111
|
+
bool, typer.Option(help="Use a simulated clock (for agents that call clock.advance).")
|
|
112
|
+
] = False,
|
|
113
|
+
max_concurrency: Annotated[int, typer.Option(min=1, help="Cases to run at once.")] = 1,
|
|
114
|
+
output: Annotated[
|
|
115
|
+
Path | None, typer.Option("--output", "-o", help="Write the full report as JSON.")
|
|
116
|
+
] = None,
|
|
117
|
+
) -> None:
|
|
118
|
+
"""Replay a dataset across model configurations and print the Pareto table."""
|
|
119
|
+
configs = _load_models(models)
|
|
120
|
+
invariants = _load_object(registry)() if registry else default_registry()
|
|
121
|
+
runner = FrontierRunner(
|
|
122
|
+
_load_object(agent),
|
|
123
|
+
invariants=invariants,
|
|
124
|
+
min_pass_rate=min_pass_rate,
|
|
125
|
+
clock_factory=SimClock if sim_clock else None,
|
|
126
|
+
max_concurrency=max_concurrency,
|
|
127
|
+
)
|
|
128
|
+
report = runner.run(load_dataset(dataset), configs)
|
|
129
|
+
typer.echo(report.to_markdown())
|
|
130
|
+
typer.echo(f"\nrecommended: {report.recommended}")
|
|
131
|
+
if output is not None:
|
|
132
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
133
|
+
output.write_text(report.model_dump_json(indent=2), encoding="utf-8")
|
|
134
|
+
typer.echo(f"report written to {output}")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
app()
|