agent-invariant-eval 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agent_eval/__init__.py ADDED
@@ -0,0 +1,63 @@
1
+ """agent-invariant-eval: deterministic, fail-closed evaluation for multi-agent workflows."""
2
+
3
+ from agent_eval._version import __version__
4
+ from agent_eval.decorators import (
5
+ EvaluationSession,
6
+ SimClock,
7
+ SpanScope,
8
+ current_session,
9
+ evaluate_agent,
10
+ )
11
+ from agent_eval.invariants import (
12
+ BaseInvariant,
13
+ InvariantRegistry,
14
+ InvariantViolation,
15
+ assert_no_context_bloat,
16
+ assert_no_pii,
17
+ assert_no_routing_loops,
18
+ assert_no_swallowed_tool_failures,
19
+ assert_numeric_grounding,
20
+ assert_pci_compliant,
21
+ assert_sla_bounds,
22
+ assert_valid_dag,
23
+ default_registry,
24
+ )
25
+ from agent_eval.schemas import (
26
+ AssertionResult,
27
+ EvalReport,
28
+ Severity,
29
+ Span,
30
+ SpanKind,
31
+ SpanStatus,
32
+ Stage,
33
+ Trajectory,
34
+ )
35
+
36
+ __all__ = [
37
+ "AssertionResult",
38
+ "BaseInvariant",
39
+ "EvalReport",
40
+ "EvaluationSession",
41
+ "InvariantRegistry",
42
+ "InvariantViolation",
43
+ "Severity",
44
+ "SimClock",
45
+ "Span",
46
+ "SpanKind",
47
+ "SpanScope",
48
+ "SpanStatus",
49
+ "Stage",
50
+ "Trajectory",
51
+ "__version__",
52
+ "assert_no_context_bloat",
53
+ "assert_no_pii",
54
+ "assert_no_routing_loops",
55
+ "assert_no_swallowed_tool_failures",
56
+ "assert_numeric_grounding",
57
+ "assert_pci_compliant",
58
+ "assert_sla_bounds",
59
+ "assert_valid_dag",
60
+ "current_session",
61
+ "default_registry",
62
+ "evaluate_agent",
63
+ ]
agent_eval/_version.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.3.0"
@@ -0,0 +1,33 @@
1
+ """Multi model replay and Pareto frontier."""
2
+
3
+ from agent_eval.benchmark.frontier import (
4
+ CaseResult,
5
+ EvalCase,
6
+ FrontierReport,
7
+ FrontierRunner,
8
+ ModelConfig,
9
+ ModelPrice,
10
+ ModelSummary,
11
+ dominates,
12
+ load_dataset,
13
+ pareto_frontier,
14
+ percentile,
15
+ recommend,
16
+ summarize,
17
+ )
18
+
19
+ __all__ = [
20
+ "CaseResult",
21
+ "EvalCase",
22
+ "FrontierReport",
23
+ "FrontierRunner",
24
+ "ModelConfig",
25
+ "ModelPrice",
26
+ "ModelSummary",
27
+ "dominates",
28
+ "load_dataset",
29
+ "pareto_frontier",
30
+ "percentile",
31
+ "recommend",
32
+ "summarize",
33
+ ]
@@ -0,0 +1,431 @@
1
+ """Cost to performance frontier.
2
+
3
+ Replays one JSONL dataset across several model configurations, scores every
4
+ run with the deterministic invariants, and reports pass rate, total inference
5
+ cost and p95 latency per model, with the Pareto optimal set and one
6
+ recommendation.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import inspect
13
+ import json
14
+ import math
15
+ from collections.abc import Callable, Iterable, Sequence
16
+ from concurrent.futures import ThreadPoolExecutor
17
+ from pathlib import Path
18
+ from typing import TYPE_CHECKING, Any, Literal
19
+
20
+ from pydantic import BaseModel, ConfigDict, Field
21
+
22
+ from agent_eval.decorators import Clock, EvaluationSession, Invariants, _as_registry
23
+ from agent_eval.invariants.base import InvariantViolation
24
+ from agent_eval.schemas import Trajectory
25
+
26
+ if TYPE_CHECKING:
27
+ from agent_eval.telemetry.otel_exporter import TelemetryEmitter
28
+
29
+
30
+ class ModelPrice(BaseModel):
31
+ """Token prices in USD per million tokens."""
32
+
33
+ model_config = ConfigDict(extra="forbid")
34
+
35
+ input_cost_per_mtok: float = Field(ge=0)
36
+ output_cost_per_mtok: float = Field(ge=0)
37
+
38
+
39
+ class ModelConfig(BaseModel):
40
+ """One configuration to benchmark. Prices are USD per million tokens.
41
+
42
+ ``input_cost_per_mtok`` and ``output_cost_per_mtok`` are the default
43
+ prices. For a configuration that mixes models (a small model for triage,
44
+ a large one for resolution) add ``pricing``, keyed by the ``model`` set on
45
+ each span. Spans whose model is not in the map use the default prices.
46
+ """
47
+
48
+ model_config = ConfigDict(extra="forbid")
49
+
50
+ name: str
51
+ input_cost_per_mtok: float = Field(ge=0)
52
+ output_cost_per_mtok: float = Field(ge=0)
53
+ pricing: dict[str, ModelPrice] = Field(default_factory=dict)
54
+ params: dict[str, Any] = Field(default_factory=dict)
55
+
56
+ def cost_usd(
57
+ self, input_tokens: float, output_tokens: float, model: str | None = None
58
+ ) -> float:
59
+ price = self.pricing.get(model) if model else None
60
+ input_price = price.input_cost_per_mtok if price else self.input_cost_per_mtok
61
+ output_price = price.output_cost_per_mtok if price else self.output_cost_per_mtok
62
+ return (input_tokens * input_price + output_tokens * output_price) / 1_000_000
63
+
64
+ def trajectory_cost_usd(self, trajectory: Trajectory) -> float:
65
+ """Exact cost of a run, priced per span and without double counting."""
66
+ return sum(
67
+ self.cost_usd(input_tokens, output_tokens, span.model)
68
+ for span, input_tokens, output_tokens in trajectory.token_attribution()
69
+ )
70
+
71
+
72
+ class EvalCase(BaseModel):
73
+ """One dataset row.
74
+
75
+ ``expect`` is the verdict a correct system produces for this case:
76
+ ``"pass"`` for work the agent should complete cleanly, ``"block"`` for
77
+ adversarial input the engine must stop. For ``"block"`` cases,
78
+ ``expected_violations`` names the invariants that must fire.
79
+ """
80
+
81
+ model_config = ConfigDict(extra="forbid")
82
+
83
+ id: str
84
+ input: Any
85
+ expect: Literal["pass", "block"] = "pass"
86
+ expected_violations: list[str] = Field(default_factory=list)
87
+ metadata: dict[str, Any] = Field(default_factory=dict)
88
+
89
+
90
+ class CaseResult(BaseModel):
91
+ case_id: str
92
+ model: str
93
+ passed: bool
94
+ verdict: Literal["pass", "block"]
95
+ expected: Literal["pass", "block"]
96
+ violations: list[str] = Field(default_factory=list)
97
+ latency_ms: float = 0.0
98
+ input_tokens: int = 0
99
+ output_tokens: int = 0
100
+ cost_usd: float = 0.0
101
+ error: str | None = None
102
+
103
+
104
+ class ModelSummary(BaseModel):
105
+ model: str
106
+ cases: int
107
+ passed: int
108
+ pass_rate: float
109
+ total_cost_usd: float
110
+ p95_latency_ms: float
111
+ mean_latency_ms: float = 0.0
112
+ total_tokens: int = 0
113
+ pareto_optimal: bool = False
114
+ recommended: bool = False
115
+
116
+
117
+ class FrontierReport(BaseModel):
118
+ min_pass_rate: float
119
+ summaries: list[ModelSummary]
120
+ results: list[CaseResult] = Field(default_factory=list)
121
+ recommended: str | None = None
122
+
123
+ def to_markdown(self) -> str:
124
+ """Recommendation table, best pass rate first."""
125
+ header = (
126
+ "| Model | Pass rate | Passed | Total cost (USD) | p95 latency (ms) | Pareto optimal "
127
+ "| Recommendation |\n"
128
+ "|---|---:|---:|---:|---:|:---:|---|\n"
129
+ )
130
+ rows = []
131
+ ordered = sorted(self.summaries, key=lambda s: (-s.pass_rate, s.total_cost_usd, s.model))
132
+ for s in ordered:
133
+ if s.recommended:
134
+ note = "**recommended**"
135
+ elif not s.pareto_optimal:
136
+ note = "dominated"
137
+ elif s.pass_rate < self.min_pass_rate:
138
+ note = f"below {self.min_pass_rate:.0%} pass floor"
139
+ else:
140
+ note = "viable"
141
+ rows.append(
142
+ f"| {s.model} | {s.pass_rate:.1%} | {s.passed}/{s.cases} | "
143
+ f"{s.total_cost_usd:.6f} | {s.p95_latency_ms:.0f} | "
144
+ f"{'yes' if s.pareto_optimal else 'no'} | {note} |"
145
+ )
146
+ return header + "\n".join(rows)
147
+
148
+
149
+ # Sync or async: may return the output or an awaitable of it.
150
+ AgentRunner = Callable[[EvalCase, ModelConfig, EvaluationSession], Any]
151
+
152
+
153
+ def load_dataset(path: str | Path) -> list[EvalCase]:
154
+ """Load a JSONL dataset. Blank lines and ``//`` comment lines are skipped."""
155
+ cases: list[EvalCase] = []
156
+ for number, line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), start=1):
157
+ stripped = line.strip()
158
+ if not stripped or stripped.startswith("//"):
159
+ continue
160
+ try:
161
+ cases.append(EvalCase.model_validate(json.loads(stripped)))
162
+ except Exception as exc:
163
+ raise ValueError(f"{path}:{number}: invalid case: {exc}") from exc
164
+ ids = [case.id for case in cases]
165
+ duplicates = sorted({i for i in ids if ids.count(i) > 1})
166
+ if duplicates:
167
+ raise ValueError(f"{path}: duplicate case ids: {', '.join(duplicates)}")
168
+ return cases
169
+
170
+
171
+ def percentile(values: Sequence[float], pct: float) -> float:
172
+ """Nearest rank percentile. ``percentile(xs, 95)`` is the p95."""
173
+ if not values:
174
+ return 0.0
175
+ if not 0 < pct <= 100:
176
+ raise ValueError("pct must be in (0, 100]")
177
+ ordered = sorted(values)
178
+ rank = math.ceil(pct / 100 * len(ordered))
179
+ return ordered[max(rank, 1) - 1]
180
+
181
+
182
+ def dominates(a: ModelSummary, b: ModelSummary) -> bool:
183
+ """True when ``a`` is at least as good as ``b`` everywhere and better somewhere.
184
+
185
+ Objectives: maximise pass rate, minimise total cost, minimise p95 latency.
186
+ """
187
+ no_worse = (
188
+ a.pass_rate >= b.pass_rate
189
+ and a.total_cost_usd <= b.total_cost_usd
190
+ and a.p95_latency_ms <= b.p95_latency_ms
191
+ )
192
+ better = (
193
+ a.pass_rate > b.pass_rate
194
+ or a.total_cost_usd < b.total_cost_usd
195
+ or a.p95_latency_ms < b.p95_latency_ms
196
+ )
197
+ return no_worse and better
198
+
199
+
200
+ def pareto_frontier(summaries: Iterable[ModelSummary]) -> list[ModelSummary]:
201
+ """The non dominated subset, in input order."""
202
+ items = list(summaries)
203
+ return [s for s in items if not any(dominates(other, s) for other in items if other is not s)]
204
+
205
+
206
+ def recommend(frontier: Sequence[ModelSummary], min_pass_rate: float) -> ModelSummary | None:
207
+ """Cheapest frontier model that clears the pass floor.
208
+
209
+ If none clears it, fall back to the highest pass rate (then cheapest), so
210
+ the table always names the least bad option.
211
+ """
212
+ if not frontier:
213
+ return None
214
+ viable = [s for s in frontier if s.pass_rate >= min_pass_rate]
215
+ if viable:
216
+ return min(
217
+ viable, key=lambda s: (s.total_cost_usd, -s.pass_rate, s.p95_latency_ms, s.model)
218
+ )
219
+ return min(frontier, key=lambda s: (-s.pass_rate, s.total_cost_usd, s.p95_latency_ms, s.model))
220
+
221
+
222
+ def summarize(
223
+ results: Iterable[CaseResult], min_pass_rate: float = 0.9
224
+ ) -> FrontierReport:
225
+ """Aggregate case results per model and mark the frontier and recommendation."""
226
+ all_results = list(results)
227
+ by_model: dict[str, list[CaseResult]] = {}
228
+ for result in all_results:
229
+ by_model.setdefault(result.model, []).append(result)
230
+
231
+ summaries: list[ModelSummary] = []
232
+ for model, rows in by_model.items():
233
+ latencies = [r.latency_ms for r in rows]
234
+ passed = sum(1 for r in rows if r.passed)
235
+ summaries.append(
236
+ ModelSummary(
237
+ model=model,
238
+ cases=len(rows),
239
+ passed=passed,
240
+ pass_rate=passed / len(rows),
241
+ total_cost_usd=round(sum(r.cost_usd for r in rows), 9),
242
+ p95_latency_ms=percentile(latencies, 95),
243
+ mean_latency_ms=round(sum(latencies) / len(latencies), 3),
244
+ total_tokens=sum(r.input_tokens + r.output_tokens for r in rows),
245
+ )
246
+ )
247
+
248
+ frontier = pareto_frontier(summaries)
249
+ for summary in frontier:
250
+ summary.pareto_optimal = True
251
+ pick = recommend(frontier, min_pass_rate)
252
+ if pick is not None:
253
+ pick.recommended = True
254
+ return FrontierReport(
255
+ min_pass_rate=min_pass_rate,
256
+ summaries=summaries,
257
+ results=all_results,
258
+ recommended=pick.model if pick else None,
259
+ )
260
+
261
+
262
+ class FrontierRunner:
263
+ """Replays a dataset across model configurations.
264
+
265
+ ``agent`` is any callable ``(case, model, session) -> output``, sync or
266
+ ``async def``. It drives the real workflow with the given configuration
267
+ and records spans on the session. The runner sets the input, captures the
268
+ output, and scores.
269
+
270
+ A case passes when the verdict equals ``case.expect`` and every name in
271
+ ``case.expected_violations`` actually fired. An agent exception is a
272
+ blocked run (fail closed).
273
+
274
+ ``max_concurrency`` runs that many cases at once: worker threads in
275
+ :meth:`run`, tasks on the caller's event loop in :meth:`arun`. Results
276
+ keep dataset order either way. Use :meth:`arun` when the agent shares
277
+ loop bound clients (one ``httpx.AsyncClient``, one SDK client).
278
+ """
279
+
280
+ def __init__(
281
+ self,
282
+ agent: AgentRunner,
283
+ *,
284
+ invariants: Invariants = None,
285
+ min_pass_rate: float = 0.9,
286
+ clock_factory: Callable[[], Clock] | None = None,
287
+ emitter: TelemetryEmitter | None = None,
288
+ max_concurrency: int = 1,
289
+ ) -> None:
290
+ if not 0.0 <= min_pass_rate <= 1.0:
291
+ raise ValueError("min_pass_rate must be between 0 and 1")
292
+ if max_concurrency < 1:
293
+ raise ValueError("max_concurrency must be >= 1")
294
+ self.agent = agent
295
+ self.registry = _as_registry(invariants)
296
+ self.min_pass_rate = min_pass_rate
297
+ self.clock_factory = clock_factory
298
+ self.emitter = emitter
299
+ self.max_concurrency = max_concurrency
300
+
301
+ # ---- one case ---------------------------------------------------------------
302
+
303
+ def _session(self, case: EvalCase, model: ModelConfig) -> EvaluationSession:
304
+ return EvaluationSession(
305
+ case.id,
306
+ invariants=self.registry,
307
+ enforce=True,
308
+ emitter=self.emitter,
309
+ clock=self.clock_factory() if self.clock_factory else None,
310
+ model=model.name,
311
+ )
312
+
313
+ @staticmethod
314
+ def _score(
315
+ case: EvalCase, model: ModelConfig, session: EvaluationSession, error: str | None
316
+ ) -> CaseResult:
317
+ report = session.report
318
+ if report is None: # defensive: no verdict means blocked
319
+ verdict: Literal["pass", "block"] = "block"
320
+ violations: list[str] = []
321
+ metrics: dict[str, float] = session.trajectory.metrics()
322
+ else:
323
+ verdict = "pass" if report.passed else "block"
324
+ violations = report.violated_names
325
+ metrics = report.metrics
326
+ matched = verdict == case.expect and set(case.expected_violations) <= set(violations)
327
+ return CaseResult(
328
+ case_id=case.id,
329
+ model=model.name,
330
+ passed=matched,
331
+ verdict=verdict,
332
+ expected=case.expect,
333
+ violations=violations,
334
+ latency_ms=float(metrics.get("latency_ms", 0.0)),
335
+ input_tokens=int(metrics.get("input_tokens", 0)),
336
+ output_tokens=int(metrics.get("output_tokens", 0)),
337
+ cost_usd=model.trajectory_cost_usd(session.trajectory),
338
+ error=error,
339
+ )
340
+
341
+ def run_case(self, case: EvalCase, model: ModelConfig) -> CaseResult:
342
+ """Run one case synchronously. An async agent gets its own event loop."""
343
+ try:
344
+ asyncio.get_running_loop()
345
+ in_event_loop = True
346
+ except RuntimeError:
347
+ in_event_loop = False
348
+
349
+ session = self._session(case, model)
350
+ error: str | None = None
351
+ misuse = False
352
+ try:
353
+ with session:
354
+ session.set_input(case.input)
355
+ output = self.agent(case, model, session)
356
+ if inspect.isawaitable(output):
357
+ if in_event_loop:
358
+ misuse = True
359
+ if inspect.iscoroutine(output):
360
+ output.close()
361
+ raise RuntimeError("async agent inside a running event loop")
362
+ output = asyncio.run(_await(output))
363
+ session.set_output(output)
364
+ except InvariantViolation:
365
+ pass
366
+ except Exception as exc: # noqa: BLE001 - recorded, the run counts as blocked
367
+ error = f"{type(exc).__name__}: {exc}"
368
+ if misuse:
369
+ raise RuntimeError(
370
+ "run() and run_case() cannot drive an async agent from inside a running "
371
+ "event loop. Use 'await runner.arun(...)' instead."
372
+ )
373
+ return self._score(case, model, session, error)
374
+
375
+ async def arun_case(self, case: EvalCase, model: ModelConfig) -> CaseResult:
376
+ """Run one case on the current event loop. A sync agent blocks the loop."""
377
+ session = self._session(case, model)
378
+ error: str | None = None
379
+ try:
380
+ with session:
381
+ session.set_input(case.input)
382
+ output = self.agent(case, model, session)
383
+ if inspect.isawaitable(output):
384
+ output = await output
385
+ session.set_output(output)
386
+ except InvariantViolation:
387
+ pass
388
+ except Exception as exc: # noqa: BLE001 - recorded, the run counts as blocked
389
+ error = f"{type(exc).__name__}: {exc}"
390
+ return self._score(case, model, session, error)
391
+
392
+ # ---- whole dataset ----------------------------------------------------------
393
+
394
+ @staticmethod
395
+ def _plan(
396
+ cases: Sequence[EvalCase], models: Sequence[ModelConfig]
397
+ ) -> list[tuple[EvalCase, ModelConfig]]:
398
+ if not cases:
399
+ raise ValueError("dataset is empty")
400
+ if not models:
401
+ raise ValueError("no model configurations given")
402
+ names = [m.name for m in models]
403
+ if len(set(names)) != len(names):
404
+ raise ValueError("model configuration names must be unique")
405
+ return [(case, model) for model in models for case in cases]
406
+
407
+ def run(self, cases: Sequence[EvalCase], models: Sequence[ModelConfig]) -> FrontierReport:
408
+ plan = self._plan(cases, models)
409
+ if self.max_concurrency == 1:
410
+ results = [self.run_case(case, model) for case, model in plan]
411
+ else:
412
+ with ThreadPoolExecutor(max_workers=self.max_concurrency) as pool:
413
+ results = list(pool.map(lambda pair: self.run_case(*pair), plan))
414
+ return summarize(results, self.min_pass_rate)
415
+
416
+ async def arun(
417
+ self, cases: Sequence[EvalCase], models: Sequence[ModelConfig]
418
+ ) -> FrontierReport:
419
+ plan = self._plan(cases, models)
420
+ gate = asyncio.Semaphore(self.max_concurrency)
421
+
422
+ async def guarded(case: EvalCase, model: ModelConfig) -> CaseResult:
423
+ async with gate:
424
+ return await self.arun_case(case, model)
425
+
426
+ results = await asyncio.gather(*(guarded(case, model) for case, model in plan))
427
+ return summarize(list(results), self.min_pass_rate)
428
+
429
+
430
+ async def _await(awaitable: Any) -> Any:
431
+ return await awaitable
agent_eval/cli.py ADDED
@@ -0,0 +1,138 @@
1
+ """Command line interface: ``agent-eval scan`` and ``agent-eval benchmark``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import Annotated, Any
10
+
11
+ try:
12
+ import typer
13
+ except ImportError as exc:
14
+ raise ImportError(
15
+ "The CLI requires the 'cli' extra: pip install 'agent-invariant-eval[cli]'"
16
+ ) from exc
17
+
18
+ from agent_eval.benchmark.frontier import FrontierRunner, ModelConfig, load_dataset
19
+ from agent_eval.decorators import SimClock
20
+ from agent_eval.invariants import default_registry, find_pci, find_pii
21
+
22
+ app = typer.Typer(
23
+ add_completion=False,
24
+ no_args_is_help=True,
25
+ help="Deterministic, fail-closed evaluation for multi-agent workflows.",
26
+ )
27
+
28
+
29
+ def _load_object(reference: str) -> Any:
30
+ """Import ``package.module:attribute``. The current directory is importable."""
31
+ module_name, _, attribute = reference.partition(":")
32
+ if not module_name or not attribute:
33
+ raise typer.BadParameter(f"expected 'module:attribute', got {reference!r}")
34
+ if str(Path.cwd()) not in sys.path:
35
+ sys.path.insert(0, str(Path.cwd()))
36
+ try:
37
+ return getattr(importlib.import_module(module_name), attribute)
38
+ except (ImportError, AttributeError) as exc:
39
+ raise typer.BadParameter(f"cannot load {reference!r}: {exc}") from exc
40
+
41
+
42
+ def _load_models(reference: str) -> list[ModelConfig]:
43
+ """Model configurations from a JSON file or a ``module:attribute`` list."""
44
+ path = Path(reference)
45
+ if path.is_file():
46
+ try:
47
+ raw = json.loads(path.read_text(encoding="utf-8"))
48
+ except json.JSONDecodeError as exc:
49
+ raise typer.BadParameter(f"{reference}: not valid JSON: {exc}") from exc
50
+ elif ":" in reference:
51
+ raw = _load_object(reference)
52
+ else:
53
+ raise typer.BadParameter(
54
+ f"{reference!r} is neither an existing JSON file nor a 'module:attribute' reference"
55
+ )
56
+ if not isinstance(raw, (list, tuple)) or not raw:
57
+ raise typer.BadParameter(f"{reference}: expected a non empty list of model configurations")
58
+ try:
59
+ return [ModelConfig.model_validate(item) for item in raw]
60
+ except ValueError as exc:
61
+ raise typer.BadParameter(f"{reference}: invalid model configuration: {exc}") from exc
62
+
63
+
64
+ @app.command()
65
+ def scan(
66
+ text: Annotated[str | None, typer.Argument(help="Text to scan. Omit to use --file.")] = None,
67
+ file: Annotated[Path | None, typer.Option("--file", "-f", help="File to scan.")] = None,
68
+ allowed_domain: Annotated[
69
+ list[str] | None,
70
+ typer.Option("--allowed-domain", help="Email domain to exempt (repeatable)."),
71
+ ] = None,
72
+ require_known_bin: Annotated[
73
+ bool,
74
+ typer.Option("--require-known-bin", help="Only flag PANs with a known card network BIN."),
75
+ ] = False,
76
+ ) -> None:
77
+ """Scan text for PII and PCI data. Findings are printed masked. Exit 1 on any finding."""
78
+ if (text is None) == (file is None):
79
+ raise typer.BadParameter("give either TEXT or --file, not both")
80
+ content = text if text is not None else file.read_text(encoding="utf-8") # type: ignore[union-attr]
81
+ findings = [
82
+ *find_pii(content, allowed_domains=allowed_domain or ()),
83
+ *find_pci(content, require_known_bin=require_known_bin),
84
+ ]
85
+ if not findings:
86
+ typer.echo("clean: no PII or PCI findings")
87
+ return
88
+ for finding in sorted(findings, key=lambda f: f.start):
89
+ typer.echo(f"{finding.kind:<20} offset {finding.start:<6} {finding.masked}")
90
+ raise typer.Exit(code=1)
91
+
92
+
93
+ @app.command()
94
+ def benchmark(
95
+ dataset: Annotated[Path, typer.Argument(help="JSONL evaluation dataset.")],
96
+ agent: Annotated[
97
+ str, typer.Option(help="Agent runner as 'module:callable' taking (case, model, session).")
98
+ ],
99
+ models: Annotated[
100
+ str,
101
+ typer.Option(
102
+ help="Model configurations: a JSON file, or 'module:attribute' naming a list."
103
+ ),
104
+ ],
105
+ registry: Annotated[
106
+ str | None,
107
+ typer.Option(help="Registry factory as 'module:callable'. Defaults to the built in set."),
108
+ ] = None,
109
+ min_pass_rate: Annotated[float, typer.Option(min=0.0, max=1.0)] = 0.9,
110
+ sim_clock: Annotated[
111
+ bool, typer.Option(help="Use a simulated clock (for agents that call clock.advance).")
112
+ ] = False,
113
+ max_concurrency: Annotated[int, typer.Option(min=1, help="Cases to run at once.")] = 1,
114
+ output: Annotated[
115
+ Path | None, typer.Option("--output", "-o", help="Write the full report as JSON.")
116
+ ] = None,
117
+ ) -> None:
118
+ """Replay a dataset across model configurations and print the Pareto table."""
119
+ configs = _load_models(models)
120
+ invariants = _load_object(registry)() if registry else default_registry()
121
+ runner = FrontierRunner(
122
+ _load_object(agent),
123
+ invariants=invariants,
124
+ min_pass_rate=min_pass_rate,
125
+ clock_factory=SimClock if sim_clock else None,
126
+ max_concurrency=max_concurrency,
127
+ )
128
+ report = runner.run(load_dataset(dataset), configs)
129
+ typer.echo(report.to_markdown())
130
+ typer.echo(f"\nrecommended: {report.recommended}")
131
+ if output is not None:
132
+ output.parent.mkdir(parents=True, exist_ok=True)
133
+ output.write_text(report.model_dump_json(indent=2), encoding="utf-8")
134
+ typer.echo(f"report written to {output}")
135
+
136
+
137
+ if __name__ == "__main__":
138
+ app()