agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,247 @@
1
+ """Reliability metrics for AgentProbe benchmarks.
2
+
3
+ Computes:
4
+ - Failure rate : fraction of runs that raised an exception
5
+ - MTTR : mean time to recovery (seconds from failure to next success)
6
+ - P50 / P90 / P99 : latency percentiles across all runs
7
+ - Recovery rate : fraction of failures from which the agent recovered
8
+ - Failure distribution: breakdown by failure category
9
+
10
+ All computations are pure Python + NumPy — no LLM calls.
11
+
12
+ Usage::
13
+
14
+ from agentprobe.benchmark.metrics import MetricsCalculator
15
+ from agentprobe.benchmark.runner import RunResult
16
+
17
+ calc = MetricsCalculator()
18
+ metrics = calc.compute(run_results)
19
+ print(metrics.failure_rate) # e.g. 0.12
20
+ print(metrics.p99_latency_ms) # e.g. 4820.0
21
+
22
+ df = metrics.to_dataframe() # pandas DataFrame for paper plots
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import statistics
28
+ from dataclasses import dataclass, field
29
+ from typing import Any
30
+
31
+ # ---------------------------------------------------------------------------
32
+ # AgentMetrics dataclass
33
+ # ---------------------------------------------------------------------------
34
+
35
+
36
+ @dataclass
37
+ class AgentMetrics:
38
+ """Aggregate reliability metrics for one (framework, task, failure_type) configuration.
39
+
40
+ All latencies are in milliseconds.
41
+ """
42
+
43
+ framework: str
44
+ task: str
45
+ failure_type: str
46
+ n_runs: int
47
+
48
+ # Execution counts
49
+ n_planned: int = 0
50
+ n_executed: int = 0
51
+ n_skipped: int = 0
52
+ condition_complete: bool = True
53
+
54
+ # Failure metrics
55
+ failure_rate: float = 0.0 # [0, 1]
56
+ n_failures: int = 0
57
+ n_successes: int = 0
58
+
59
+ # Recovery metrics
60
+ recovery_rate: float | None = None # None when no failures; fraction otherwise
61
+ mttr_ms: float = 0.0 # mean time to recovery (ms)
62
+ mttr_ms_std: float = 0.0
63
+
64
+ # Latency percentiles (total turn latency including injected delays)
65
+ mean_latency_ms: float = 0.0
66
+ std_latency_ms: float = 0.0
67
+ p50_latency_ms: float = 0.0
68
+ p90_latency_ms: float = 0.0
69
+ p99_latency_ms: float = 0.0
70
+ max_latency_ms: float = 0.0
71
+
72
+ # Failure breakdown by category
73
+ failure_by_category: dict[str, int] = field(default_factory=dict)
74
+
75
+ def to_dict(self) -> dict[str, Any]:
76
+ return {
77
+ "framework": self.framework,
78
+ "task": self.task,
79
+ "failure_type": self.failure_type,
80
+ "n_runs": self.n_runs,
81
+ "n_planned": self.n_planned,
82
+ "n_executed": self.n_executed,
83
+ "n_skipped": self.n_skipped,
84
+ "condition_complete": self.condition_complete,
85
+ "failure_rate": round(self.failure_rate, 4),
86
+ "n_failures": self.n_failures,
87
+ "n_successes": self.n_successes,
88
+ "recovery_rate": round(self.recovery_rate, 4)
89
+ if self.recovery_rate is not None
90
+ else None,
91
+ "mttr_ms": round(self.mttr_ms, 2),
92
+ "mean_latency_ms": round(self.mean_latency_ms, 2),
93
+ "p50_latency_ms": round(self.p50_latency_ms, 2),
94
+ "p90_latency_ms": round(self.p90_latency_ms, 2),
95
+ "p99_latency_ms": round(self.p99_latency_ms, 2),
96
+ "max_latency_ms": round(self.max_latency_ms, 2),
97
+ }
98
+
99
+ def to_dataframe_row(self) -> dict[str, Any]:
100
+ return self.to_dict()
101
+
102
+ def __str__(self) -> str:
103
+ return (
104
+ f"AgentMetrics({self.framework}/{self.task}/{self.failure_type}) "
105
+ f"failure_rate={self.failure_rate:.1%} "
106
+ f"mttr={self.mttr_ms:.0f}ms "
107
+ f"p99={self.p99_latency_ms:.0f}ms"
108
+ )
109
+
110
+
111
+ # ---------------------------------------------------------------------------
112
+ # Calculator
113
+ # ---------------------------------------------------------------------------
114
+
115
+
116
+ class MetricsCalculator:
117
+ """Computes AgentProbe reliability metrics from a list of RunResult objects."""
118
+
119
+ def compute(
120
+ self, runs: list[Any], framework: str = "", task: str = "", failure_type: str = ""
121
+ ) -> AgentMetrics:
122
+ """Compute metrics from a list of :class:`~agentprobe.benchmark.runner.RunResult`.
123
+
124
+ Args:
125
+ runs: List of RunResult objects.
126
+ framework: Framework name (used for labeling; inferred from runs if empty).
127
+ task: Task name (used for labeling; inferred from runs if empty).
128
+ failure_type: Failure type (used for labeling; inferred from runs if empty).
129
+ """
130
+ if not runs:
131
+ return AgentMetrics(framework=framework, task=task, failure_type=failure_type, n_runs=0)
132
+
133
+ # Infer labels from first run if not provided
134
+ first = runs[0]
135
+ _g = lambda obj, key, default="": (
136
+ obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)
137
+ ) # noqa: E731
138
+ fw = framework or _g(first, "framework", "unknown")
139
+ tk = task or _g(first, "task", "unknown")
140
+ ft = failure_type or _g(first, "failure_type", "none")
141
+
142
+ def _get(obj: Any, key: str, default: Any = None) -> Any:
143
+ """Get attribute from object or dict."""
144
+ if isinstance(obj, dict):
145
+ return obj.get(key, default)
146
+ return getattr(obj, key, default)
147
+
148
+ # Separate executed from skipped trials
149
+ n_planned = len(runs)
150
+ executed = [r for r in runs if _get(r, "status", "completed") == "completed"]
151
+ n_skipped = n_planned - len(executed)
152
+ condition_complete = n_skipped == 0
153
+
154
+ # Compute metrics only from executed trials
155
+ successes = [r for r in executed if _get(r, "success", False)]
156
+ failures = [r for r in executed if not _get(r, "success", False)]
157
+ recovered = [r for r in failures if _get(r, "recovered", False)]
158
+
159
+ latencies = [_get(r, "latency_ms", 0.0) for r in executed]
160
+ recovery_latencies = [_get(r, "recovery_latency_ms", 0.0) for r in recovered]
161
+
162
+ n = len(executed)
163
+ failure_rate = len(failures) / n if n > 0 else 0.0
164
+ recovery_rate = len(recovered) / len(failures) if failures else None
165
+
166
+ return AgentMetrics(
167
+ framework=fw,
168
+ task=tk,
169
+ failure_type=ft,
170
+ n_runs=n,
171
+ n_planned=n_planned,
172
+ n_executed=n,
173
+ n_skipped=n_skipped,
174
+ condition_complete=condition_complete,
175
+ failure_rate=failure_rate,
176
+ n_failures=len(failures),
177
+ n_successes=len(successes),
178
+ recovery_rate=recovery_rate,
179
+ mttr_ms=statistics.mean(recovery_latencies) if recovery_latencies else 0.0,
180
+ mttr_ms_std=statistics.stdev(recovery_latencies)
181
+ if len(recovery_latencies) > 1
182
+ else 0.0,
183
+ mean_latency_ms=statistics.mean(latencies) if latencies else 0.0,
184
+ std_latency_ms=statistics.stdev(latencies) if len(latencies) > 1 else 0.0,
185
+ p50_latency_ms=_percentile(latencies, 50),
186
+ p90_latency_ms=_percentile(latencies, 90),
187
+ p99_latency_ms=_percentile(latencies, 99),
188
+ max_latency_ms=max(latencies) if latencies else 0.0,
189
+ failure_by_category=self._count_by_category(failures),
190
+ )
191
+
192
+ def compute_all(self, results_by_config: dict[str, list[Any]]) -> list[AgentMetrics]:
193
+ """Compute metrics for multiple configurations.
194
+
195
+ Args:
196
+ results_by_config: Maps config label (e.g. "langgraph/rag/timeout")
197
+ to a list of RunResult objects.
198
+ """
199
+ metrics = []
200
+ for config_label, runs in results_by_config.items():
201
+ parts = config_label.split("/")
202
+ fw = parts[0] if len(parts) > 0 else ""
203
+ tk = parts[1] if len(parts) > 1 else ""
204
+ ft = parts[2] if len(parts) > 2 else ""
205
+ metrics.append(self.compute(runs, framework=fw, task=tk, failure_type=ft))
206
+ return metrics
207
+
208
+ def to_dataframe(self, metrics_list: list[AgentMetrics]) -> Any:
209
+ """Convert a list of AgentMetrics to a pandas DataFrame."""
210
+ try:
211
+ import pandas as pd
212
+ except ImportError as e:
213
+ raise ImportError("pandas is required for to_dataframe(). pip install pandas") from e
214
+ rows = [m.to_dict() for m in metrics_list]
215
+ return pd.DataFrame(rows)
216
+
217
+ def _count_by_category(self, failures: list[Any]) -> dict[str, int]:
218
+ counts: dict[str, int] = {}
219
+ for r in failures:
220
+ cat = (
221
+ r.get("failure_category", "unknown")
222
+ if isinstance(r, dict)
223
+ else getattr(r, "failure_category", "unknown")
224
+ )
225
+ counts[cat] = counts.get(cat, 0) + 1
226
+ return counts
227
+
228
+
229
+ # ---------------------------------------------------------------------------
230
+ # Helpers
231
+ # ---------------------------------------------------------------------------
232
+
233
+
234
+ def _percentile(data: list[float], pct: int) -> float:
235
+ """Ceiling nearest-rank percentile.
236
+
237
+ For P99 at n=30, ceil(30 * 99/100) = 30, so this returns the sample
238
+ maximum — consistent with the paper's convention that P99 equals the
239
+ empirical sample maximum for n < 100.
240
+ """
241
+ if not data:
242
+ return 0.0
243
+ sorted_data = sorted(data)
244
+ import math
245
+
246
+ idx = min(math.ceil(len(sorted_data) * pct / 100), len(sorted_data)) - 1
247
+ return sorted_data[max(0, idx)]
@@ -0,0 +1,434 @@
1
+ """Benchmark runner for AgentProbe.
2
+
3
+ Orchestrates multi-run experiments across agent frameworks, failure types,
4
+ and tasks. Produces structured RunResult records used by MetricsCalculator.
5
+
6
+ Usage::
7
+
8
+ from agentprobe.benchmark.runner import BenchmarkRunner, BenchmarkConfig
9
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
10
+ from agentprobe.agents.langgraph_agent import LangGraphToolAgent
11
+
12
+ config = BenchmarkConfig(
13
+ framework="langgraph",
14
+ task="tool_use",
15
+ n_runs=50,
16
+ failure_config=ToolFailureConfig(
17
+ failure_type=ToolFailureType.TIMEOUT,
18
+ failure_probability=0.2,
19
+ ),
20
+ )
21
+
22
+ agent = LangGraphToolAgent(llm=my_llm)
23
+ runner = BenchmarkRunner(agent=agent, config=config)
24
+ results = runner.run()
25
+
26
+ from agentprobe.benchmark.metrics import MetricsCalculator
27
+ metrics = MetricsCalculator().compute(results)
28
+ print(metrics)
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import concurrent.futures
34
+ import csv
35
+ import json
36
+ import time
37
+ from collections.abc import Callable
38
+ from dataclasses import dataclass, field
39
+ from pathlib import Path
40
+ from typing import TYPE_CHECKING, Any
41
+
42
+ from opentelemetry import context as otel_context
43
+ from rich.console import Console
44
+ from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
45
+
46
+ from agentprobe.benchmark.metrics import _percentile
47
+ from agentprobe.observer.classifier import FailureClassifier
48
+ from agentprobe.observer.tracer import get_tracer
49
+
50
+ if TYPE_CHECKING:
51
+ from agentprobe.injector.tool_failure import ToolFailureConfig
52
+
53
+ _tracer = get_tracer(__name__)
54
+ _console = Console()
55
+
56
+
57
+ class _HarnessTimeout(Exception):
58
+ """Raised when the benchmark runner's wall-clock deadline is exceeded.
59
+
60
+ Distinct from ``TimeoutError`` so that upstream timeouts (e.g. pyautogen's
61
+ ``Completion.request_timeout``) are not misclassified as harness deadlines.
62
+ """
63
+
64
+
65
+ # ---------------------------------------------------------------------------
66
+ # Data structures
67
+ # ---------------------------------------------------------------------------
68
+
69
+
70
+ @dataclass
71
+ class RunResult:
72
+ """Result of a single benchmark run (one agent invocation).
73
+
74
+ The ``status`` field distinguishes executed trials from those skipped
75
+ after a harness timeout aborted the condition:
76
+ - ``"completed"`` — trial was dispatched and finished (success or failure).
77
+ - ``"skipped"`` — trial was never dispatched (prior timeout lingering).
78
+
79
+ Only ``"completed"`` trials should be included in error-rate, recovery,
80
+ and latency calculations.
81
+ """
82
+
83
+ run_id: int
84
+ framework: str
85
+ task: str
86
+ failure_type: str
87
+ success: bool
88
+ latency_ms: float
89
+ status: str = "completed"
90
+ failure_category: str = "none"
91
+ exception_type: str = ""
92
+ exception_msg: str = ""
93
+ recovered: bool = False
94
+ recovery_latency_ms: float = 0.0
95
+ recovery_output: str = ""
96
+ output: str = ""
97
+ metadata: dict[str, Any] = field(default_factory=dict)
98
+
99
+ def to_dict(self) -> dict[str, Any]:
100
+ return {
101
+ "run_id": self.run_id,
102
+ "framework": self.framework,
103
+ "task": self.task,
104
+ "failure_type": self.failure_type,
105
+ "status": self.status,
106
+ "success": self.success,
107
+ "latency_ms": round(self.latency_ms, 2),
108
+ "failure_category": self.failure_category,
109
+ "exception_type": self.exception_type,
110
+ "exception_msg": self.exception_msg,
111
+ "recovered": self.recovered,
112
+ "recovery_latency_ms": round(self.recovery_latency_ms, 2),
113
+ "recovery_output": self.recovery_output,
114
+ "output": self.output,
115
+ }
116
+
117
+
118
+ @dataclass
119
+ class BenchmarkConfig:
120
+ """Configuration for one benchmark experiment.
121
+
122
+ Attributes:
123
+ framework: Agent framework identifier (e.g. "langgraph", "langchain").
124
+ task: Task identifier (e.g. "tool_use", "rag", "multi_agent").
125
+ n_runs: Number of independent runs per configuration.
126
+ failure_config: Tool failure injection configuration.
127
+ If None, runs without any injection (baseline).
128
+ recovery_attempts: How many times to retry after a failure.
129
+ Set to 0 to measure failure rate without recovery.
130
+ timeout_per_run_seconds: Wall-clock timeout per run.
131
+ seed: Global RNG seed for reproducibility.
132
+ save_results_path: If set, writes CSV/JSON results to this path.
133
+ """
134
+
135
+ framework: str = "langgraph"
136
+ task: str = "tool_use"
137
+ n_runs: int = 30
138
+ failure_config: ToolFailureConfig | None = None
139
+ recovery_attempts: int = 1
140
+ timeout_per_run_seconds: float = 60.0
141
+ seed: int = 42
142
+ save_results_path: str | None = None
143
+
144
+
145
+ # ---------------------------------------------------------------------------
146
+ # Runner
147
+ # ---------------------------------------------------------------------------
148
+
149
+
150
+ class BenchmarkRunner:
151
+ """Runs benchmark experiments for one (framework, task, failure_config) triple.
152
+
153
+ Args:
154
+ agent: Any object satisfying :class:`agentprobe.agents.BaseAgent`
155
+ (``invoke(input: dict) -> dict`` plus ``reset()``), e.g. the
156
+ LangGraph, LangChain, or AutoGen adapters in ``agentprobe.agents``.
157
+ config: Benchmark configuration.
158
+ task_fn: Optional callable that generates task inputs per run.
159
+ Signature: ``(run_id: int, task: str) -> dict``.
160
+ Defaults to :func:`paper_task_fn`, the five-prompt set used in the
161
+ paper. Use ``agentprobe.benchmark.tasks.corpus_task_fn(task)`` to
162
+ run against the larger graded corpora instead.
163
+ """
164
+
165
+ def __init__(
166
+ self,
167
+ agent: Any,
168
+ config: BenchmarkConfig | None = None,
169
+ task_fn: Callable[[int, str], dict] | None = None,
170
+ ) -> None:
171
+ self.agent = agent
172
+ self.config = config or BenchmarkConfig()
173
+ self._task_fn = task_fn or paper_task_fn
174
+ self._classifier: Any = FailureClassifier()
175
+ self._invocation_hung = False
176
+
177
+ def run(self) -> list[RunResult]:
178
+ """Execute the full benchmark. Returns a list of RunResult objects."""
179
+ cfg = self.config
180
+ failure_type = cfg.failure_config.failure_type.value if cfg.failure_config else "none"
181
+
182
+ _console.rule(
183
+ f"[bold blue]AgentProbe Benchmark — {cfg.framework}/{cfg.task}/{failure_type}"
184
+ )
185
+ _console.print(f" Runs: {cfg.n_runs} | Recovery attempts: {cfg.recovery_attempts}")
186
+
187
+ results: list[RunResult] = []
188
+
189
+ with Progress(
190
+ SpinnerColumn(),
191
+ "[progress.description]{task.description}",
192
+ TimeElapsedColumn(),
193
+ console=_console,
194
+ ) as progress:
195
+ task_bar = progress.add_task(f"[cyan]Running {cfg.n_runs} trials...", total=cfg.n_runs)
196
+
197
+ for run_id in range(cfg.n_runs):
198
+ if self._invocation_hung:
199
+ results.append(
200
+ RunResult(
201
+ run_id=run_id,
202
+ framework=cfg.framework,
203
+ task=cfg.task,
204
+ failure_type=failure_type,
205
+ success=False,
206
+ latency_ms=0.0,
207
+ status="skipped",
208
+ failure_category="harness_abort",
209
+ exception_type="_HarnessTimeout",
210
+ exception_msg="Skipped: prior trial timed out with lingering thread",
211
+ )
212
+ )
213
+ progress.advance(task_bar)
214
+ continue
215
+ result = self._run_single(run_id, failure_type)
216
+ results.append(result)
217
+ progress.advance(task_bar)
218
+
219
+ _console.print(self._summary_line(results))
220
+
221
+ if cfg.save_results_path:
222
+ self._save(results, cfg.save_results_path)
223
+
224
+ return results
225
+
226
+ # ------------------------------------------------------------------
227
+ # Internal
228
+ # ------------------------------------------------------------------
229
+
230
+ def _run_single(self, run_id: int, failure_type: str) -> RunResult:
231
+ cfg = self.config
232
+ task_input = self._task_fn(run_id, cfg.task)
233
+
234
+ with _tracer.start_as_current_span(
235
+ f"benchmark.run.{run_id}",
236
+ attributes={
237
+ "run.id": run_id,
238
+ "framework": cfg.framework,
239
+ "task": cfg.task,
240
+ "failure_type": failure_type,
241
+ },
242
+ ):
243
+ start = time.perf_counter()
244
+ try:
245
+ output = self._invoke_with_timeout(task_input)
246
+ latency_ms = (time.perf_counter() - start) * 1000
247
+ return RunResult(
248
+ run_id=run_id,
249
+ framework=cfg.framework,
250
+ task=cfg.task,
251
+ failure_type=failure_type,
252
+ success=True,
253
+ latency_ms=latency_ms,
254
+ output=str(output),
255
+ )
256
+ except Exception as exc:
257
+ latency_ms = (time.perf_counter() - start) * 1000
258
+ category = "unknown"
259
+ if self._classifier is not None:
260
+ event = self._classifier.classify_exception(exc, framework=cfg.framework)
261
+ category = event.category.value
262
+
263
+ # Attempt recovery
264
+ recovered, recovery_latency_ms, recovery_output = self._attempt_recovery(task_input)
265
+
266
+ return RunResult(
267
+ run_id=run_id,
268
+ framework=cfg.framework,
269
+ task=cfg.task,
270
+ failure_type=failure_type,
271
+ success=False,
272
+ latency_ms=latency_ms,
273
+ failure_category=category,
274
+ exception_type=type(exc).__name__,
275
+ exception_msg=str(exc)[:200],
276
+ recovered=recovered,
277
+ recovery_latency_ms=recovery_latency_ms,
278
+ recovery_output=str(recovery_output) if recovery_output else "",
279
+ )
280
+
281
+ def _invoke_with_timeout(self, task_input: dict) -> Any:
282
+ """Invoke agent with wall-clock timeout enforcement.
283
+
284
+ Uses a thread so that the agent's injector RNG, injection logs,
285
+ and all mutable state advance in-process (no fork/copy issues).
286
+ The current OpenTelemetry context is explicitly propagated into the
287
+ worker thread so that agent spans become children of the benchmark span.
288
+
289
+ Guarantees:
290
+ 1. Upstream exceptions (including ``TimeoutError``) are propagated
291
+ with their original type and message.
292
+ 2. Only genuine wall-clock deadline expiry raises ``_HarnessTimeout``.
293
+ 3. After a harness timeout, ``self._invocation_hung`` is set to
294
+ ``True`` so that ``_attempt_recovery`` skips retry and
295
+ ``run()`` aborts remaining trials, preventing overlap between
296
+ the lingering thread and new invocations.
297
+
298
+ Limitation: Python threads cannot be forcibly killed. A hung
299
+ thread may linger until the process exits.
300
+ """
301
+ timeout = self.config.timeout_per_run_seconds
302
+ ctx = otel_context.get_current()
303
+
304
+ def _worker() -> Any:
305
+ token = otel_context.attach(ctx)
306
+ try:
307
+ return self.agent.invoke(task_input)
308
+ finally:
309
+ otel_context.detach(token)
310
+
311
+ pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
312
+ future = pool.submit(_worker)
313
+ pool.shutdown(wait=False)
314
+
315
+ try:
316
+ return future.result(timeout=timeout)
317
+ except Exception:
318
+ if future.done():
319
+ # Worker finished (possibly with an exception) before
320
+ # the deadline — propagate the original exception as-is.
321
+ return future.result() # re-raises the worker exception
322
+ # Worker still running — genuine deadline miss.
323
+ self._invocation_hung = True
324
+ raise _HarnessTimeout(f"Agent invocation exceeded {timeout}s wall-clock timeout")
325
+
326
+ def _attempt_recovery(self, task_input: dict) -> tuple[bool, float, Any]:
327
+ """Try to invoke the agent again after a failure.
328
+
329
+ If the prior invocation is still running (hung thread after a
330
+ harness timeout), recovery is skipped entirely to prevent two
331
+ concurrent invocations on the same agent.
332
+
333
+ Returns (recovered, latency_ms, output).
334
+ """
335
+ if self.config.recovery_attempts == 0:
336
+ return False, 0.0, None
337
+ if self._invocation_hung:
338
+ return False, 0.0, None
339
+ for _ in range(self.config.recovery_attempts):
340
+ start = time.perf_counter()
341
+ try:
342
+ output = self._invoke_with_timeout(task_input)
343
+ return True, (time.perf_counter() - start) * 1000, output
344
+ except Exception:
345
+ if self._invocation_hung:
346
+ break # recovery attempt itself timed out; stop retrying
347
+ continue
348
+ return False, 0.0, None
349
+
350
+ def _summary_line(self, results: list[RunResult]) -> str:
351
+ executed = [r for r in results if r.status == "completed"]
352
+ skipped = len(results) - len(executed)
353
+ total = len(executed)
354
+ failures = sum(1 for r in executed if not r.success)
355
+ rate = failures / total if total > 0 else 0.0
356
+ recovered = sum(1 for r in executed if r.recovered)
357
+ latencies = [r.latency_ms for r in executed]
358
+ p99 = _percentile(latencies, 99)
359
+ line = (
360
+ f"[green]Done[/green] — {total} executed | "
361
+ f"failure_rate={rate:.1%} | "
362
+ f"recovered={recovered}/{failures} | "
363
+ f"P99={p99:.0f}ms"
364
+ )
365
+ if skipped:
366
+ line += f" | [yellow]{skipped} skipped (condition incomplete)[/yellow]"
367
+ return line
368
+
369
+ def _save(self, results: list[RunResult], path: str) -> None:
370
+ p = Path(path)
371
+ p.parent.mkdir(parents=True, exist_ok=True)
372
+
373
+ # CSV
374
+ csv_path = p.with_suffix(".csv")
375
+ with open(csv_path, "w", newline="") as f:
376
+ if results:
377
+ writer = csv.DictWriter(f, fieldnames=results[0].to_dict().keys())
378
+ writer.writeheader()
379
+ writer.writerows(r.to_dict() for r in results)
380
+
381
+ # JSON
382
+ json_path = p.with_suffix(".json")
383
+ with open(json_path, "w") as f:
384
+ json.dump([r.to_dict() for r in results], f, indent=2)
385
+
386
+ _console.print(f"[dim]Results saved to {csv_path} and {json_path}[/dim]")
387
+
388
+
389
+ # ---------------------------------------------------------------------------
390
+ # Default task generator — the paper's prompt set
391
+ # ---------------------------------------------------------------------------
392
+
393
+ #: The five prompts per task type used in every experiment reported in the paper
394
+ #: (Phase 1, Phase 2, and cross-model validation). Run ``i`` receives prompt
395
+ #: ``i % 5``. Kept verbatim so that published batches reproduce exactly.
396
+ #: The larger graded corpora in :mod:`agentprobe.benchmark.tasks` are *not* the
397
+ #: paper's prompts; use ``corpus_task_fn()`` from that module to benchmark
398
+ #: against them instead.
399
+ PAPER_PROMPTS: dict[str, list[str]] = {
400
+ "tool_use": [
401
+ "What is the capital of France?",
402
+ "Search for recent news about LLM reliability.",
403
+ "Calculate 15% of 847.",
404
+ "What is the current weather in San Francisco?",
405
+ "Find the top 3 papers on multi-agent systems published in 2024.",
406
+ ],
407
+ "rag": [
408
+ "Summarize the main contributions of the AgentProbe paper.",
409
+ "What are the 5 failure categories in the AgentProbe taxonomy?",
410
+ "How does AgentProbe measure MTTR?",
411
+ "Compare LangGraph and LangChain reliability profiles.",
412
+ "What observability gaps exist in current agentic frameworks?",
413
+ ],
414
+ "multi_agent": [
415
+ "Research and then summarize the latest advances in LLM agents.",
416
+ "Find relevant papers and write a literature review outline.",
417
+ "Analyze the dataset and produce a visualization plan.",
418
+ "Draft a blog post about AI reliability best practices.",
419
+ "Create a benchmark plan for testing agent robustness.",
420
+ ],
421
+ }
422
+
423
+
424
+ def paper_task_fn(run_id: int, task: str) -> dict[str, Any]:
425
+ """Default ``task_fn``: cycle through :data:`PAPER_PROMPTS` for *task*.
426
+
427
+ Unknown task names fall back to the ``tool_use`` prompts.
428
+ """
429
+ inputs = PAPER_PROMPTS.get(task, PAPER_PROMPTS["tool_use"])
430
+ return {"input": inputs[run_id % len(inputs)], "run_id": run_id}
431
+
432
+
433
+ # Backwards-compatible alias used by the experiment scripts.
434
+ _default_task_fn = paper_task_fn