agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Reliability metrics for AgentProbe benchmarks.
|
|
2
|
+
|
|
3
|
+
Computes:
|
|
4
|
+
- Failure rate : fraction of runs that raised an exception
|
|
5
|
+
- MTTR : mean time to recovery (seconds from failure to next success)
|
|
6
|
+
- P50 / P90 / P99 : latency percentiles across all runs
|
|
7
|
+
- Recovery rate : fraction of failures from which the agent recovered
|
|
8
|
+
- Failure distribution: breakdown by failure category
|
|
9
|
+
|
|
10
|
+
All computations are pure Python + NumPy — no LLM calls.
|
|
11
|
+
|
|
12
|
+
Usage::
|
|
13
|
+
|
|
14
|
+
from agentprobe.benchmark.metrics import MetricsCalculator
|
|
15
|
+
from agentprobe.benchmark.runner import RunResult
|
|
16
|
+
|
|
17
|
+
calc = MetricsCalculator()
|
|
18
|
+
metrics = calc.compute(run_results)
|
|
19
|
+
print(metrics.failure_rate) # e.g. 0.12
|
|
20
|
+
print(metrics.p99_latency_ms) # e.g. 4820.0
|
|
21
|
+
|
|
22
|
+
df = metrics.to_dataframe() # pandas DataFrame for paper plots
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import statistics
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
# AgentMetrics dataclass
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class AgentMetrics:
|
|
38
|
+
"""Aggregate reliability metrics for one (framework, task, failure_type) configuration.
|
|
39
|
+
|
|
40
|
+
All latencies are in milliseconds.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
framework: str
|
|
44
|
+
task: str
|
|
45
|
+
failure_type: str
|
|
46
|
+
n_runs: int
|
|
47
|
+
|
|
48
|
+
# Execution counts
|
|
49
|
+
n_planned: int = 0
|
|
50
|
+
n_executed: int = 0
|
|
51
|
+
n_skipped: int = 0
|
|
52
|
+
condition_complete: bool = True
|
|
53
|
+
|
|
54
|
+
# Failure metrics
|
|
55
|
+
failure_rate: float = 0.0 # [0, 1]
|
|
56
|
+
n_failures: int = 0
|
|
57
|
+
n_successes: int = 0
|
|
58
|
+
|
|
59
|
+
# Recovery metrics
|
|
60
|
+
recovery_rate: float | None = None # None when no failures; fraction otherwise
|
|
61
|
+
mttr_ms: float = 0.0 # mean time to recovery (ms)
|
|
62
|
+
mttr_ms_std: float = 0.0
|
|
63
|
+
|
|
64
|
+
# Latency percentiles (total turn latency including injected delays)
|
|
65
|
+
mean_latency_ms: float = 0.0
|
|
66
|
+
std_latency_ms: float = 0.0
|
|
67
|
+
p50_latency_ms: float = 0.0
|
|
68
|
+
p90_latency_ms: float = 0.0
|
|
69
|
+
p99_latency_ms: float = 0.0
|
|
70
|
+
max_latency_ms: float = 0.0
|
|
71
|
+
|
|
72
|
+
# Failure breakdown by category
|
|
73
|
+
failure_by_category: dict[str, int] = field(default_factory=dict)
|
|
74
|
+
|
|
75
|
+
def to_dict(self) -> dict[str, Any]:
|
|
76
|
+
return {
|
|
77
|
+
"framework": self.framework,
|
|
78
|
+
"task": self.task,
|
|
79
|
+
"failure_type": self.failure_type,
|
|
80
|
+
"n_runs": self.n_runs,
|
|
81
|
+
"n_planned": self.n_planned,
|
|
82
|
+
"n_executed": self.n_executed,
|
|
83
|
+
"n_skipped": self.n_skipped,
|
|
84
|
+
"condition_complete": self.condition_complete,
|
|
85
|
+
"failure_rate": round(self.failure_rate, 4),
|
|
86
|
+
"n_failures": self.n_failures,
|
|
87
|
+
"n_successes": self.n_successes,
|
|
88
|
+
"recovery_rate": round(self.recovery_rate, 4)
|
|
89
|
+
if self.recovery_rate is not None
|
|
90
|
+
else None,
|
|
91
|
+
"mttr_ms": round(self.mttr_ms, 2),
|
|
92
|
+
"mean_latency_ms": round(self.mean_latency_ms, 2),
|
|
93
|
+
"p50_latency_ms": round(self.p50_latency_ms, 2),
|
|
94
|
+
"p90_latency_ms": round(self.p90_latency_ms, 2),
|
|
95
|
+
"p99_latency_ms": round(self.p99_latency_ms, 2),
|
|
96
|
+
"max_latency_ms": round(self.max_latency_ms, 2),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
def to_dataframe_row(self) -> dict[str, Any]:
|
|
100
|
+
return self.to_dict()
|
|
101
|
+
|
|
102
|
+
def __str__(self) -> str:
|
|
103
|
+
return (
|
|
104
|
+
f"AgentMetrics({self.framework}/{self.task}/{self.failure_type}) "
|
|
105
|
+
f"failure_rate={self.failure_rate:.1%} "
|
|
106
|
+
f"mttr={self.mttr_ms:.0f}ms "
|
|
107
|
+
f"p99={self.p99_latency_ms:.0f}ms"
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
# Calculator
|
|
113
|
+
# ---------------------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class MetricsCalculator:
|
|
117
|
+
"""Computes AgentProbe reliability metrics from a list of RunResult objects."""
|
|
118
|
+
|
|
119
|
+
def compute(
|
|
120
|
+
self, runs: list[Any], framework: str = "", task: str = "", failure_type: str = ""
|
|
121
|
+
) -> AgentMetrics:
|
|
122
|
+
"""Compute metrics from a list of :class:`~agentprobe.benchmark.runner.RunResult`.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
runs: List of RunResult objects.
|
|
126
|
+
framework: Framework name (used for labeling; inferred from runs if empty).
|
|
127
|
+
task: Task name (used for labeling; inferred from runs if empty).
|
|
128
|
+
failure_type: Failure type (used for labeling; inferred from runs if empty).
|
|
129
|
+
"""
|
|
130
|
+
if not runs:
|
|
131
|
+
return AgentMetrics(framework=framework, task=task, failure_type=failure_type, n_runs=0)
|
|
132
|
+
|
|
133
|
+
# Infer labels from first run if not provided
|
|
134
|
+
first = runs[0]
|
|
135
|
+
_g = lambda obj, key, default="": (
|
|
136
|
+
obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)
|
|
137
|
+
) # noqa: E731
|
|
138
|
+
fw = framework or _g(first, "framework", "unknown")
|
|
139
|
+
tk = task or _g(first, "task", "unknown")
|
|
140
|
+
ft = failure_type or _g(first, "failure_type", "none")
|
|
141
|
+
|
|
142
|
+
def _get(obj: Any, key: str, default: Any = None) -> Any:
|
|
143
|
+
"""Get attribute from object or dict."""
|
|
144
|
+
if isinstance(obj, dict):
|
|
145
|
+
return obj.get(key, default)
|
|
146
|
+
return getattr(obj, key, default)
|
|
147
|
+
|
|
148
|
+
# Separate executed from skipped trials
|
|
149
|
+
n_planned = len(runs)
|
|
150
|
+
executed = [r for r in runs if _get(r, "status", "completed") == "completed"]
|
|
151
|
+
n_skipped = n_planned - len(executed)
|
|
152
|
+
condition_complete = n_skipped == 0
|
|
153
|
+
|
|
154
|
+
# Compute metrics only from executed trials
|
|
155
|
+
successes = [r for r in executed if _get(r, "success", False)]
|
|
156
|
+
failures = [r for r in executed if not _get(r, "success", False)]
|
|
157
|
+
recovered = [r for r in failures if _get(r, "recovered", False)]
|
|
158
|
+
|
|
159
|
+
latencies = [_get(r, "latency_ms", 0.0) for r in executed]
|
|
160
|
+
recovery_latencies = [_get(r, "recovery_latency_ms", 0.0) for r in recovered]
|
|
161
|
+
|
|
162
|
+
n = len(executed)
|
|
163
|
+
failure_rate = len(failures) / n if n > 0 else 0.0
|
|
164
|
+
recovery_rate = len(recovered) / len(failures) if failures else None
|
|
165
|
+
|
|
166
|
+
return AgentMetrics(
|
|
167
|
+
framework=fw,
|
|
168
|
+
task=tk,
|
|
169
|
+
failure_type=ft,
|
|
170
|
+
n_runs=n,
|
|
171
|
+
n_planned=n_planned,
|
|
172
|
+
n_executed=n,
|
|
173
|
+
n_skipped=n_skipped,
|
|
174
|
+
condition_complete=condition_complete,
|
|
175
|
+
failure_rate=failure_rate,
|
|
176
|
+
n_failures=len(failures),
|
|
177
|
+
n_successes=len(successes),
|
|
178
|
+
recovery_rate=recovery_rate,
|
|
179
|
+
mttr_ms=statistics.mean(recovery_latencies) if recovery_latencies else 0.0,
|
|
180
|
+
mttr_ms_std=statistics.stdev(recovery_latencies)
|
|
181
|
+
if len(recovery_latencies) > 1
|
|
182
|
+
else 0.0,
|
|
183
|
+
mean_latency_ms=statistics.mean(latencies) if latencies else 0.0,
|
|
184
|
+
std_latency_ms=statistics.stdev(latencies) if len(latencies) > 1 else 0.0,
|
|
185
|
+
p50_latency_ms=_percentile(latencies, 50),
|
|
186
|
+
p90_latency_ms=_percentile(latencies, 90),
|
|
187
|
+
p99_latency_ms=_percentile(latencies, 99),
|
|
188
|
+
max_latency_ms=max(latencies) if latencies else 0.0,
|
|
189
|
+
failure_by_category=self._count_by_category(failures),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def compute_all(self, results_by_config: dict[str, list[Any]]) -> list[AgentMetrics]:
|
|
193
|
+
"""Compute metrics for multiple configurations.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
results_by_config: Maps config label (e.g. "langgraph/rag/timeout")
|
|
197
|
+
to a list of RunResult objects.
|
|
198
|
+
"""
|
|
199
|
+
metrics = []
|
|
200
|
+
for config_label, runs in results_by_config.items():
|
|
201
|
+
parts = config_label.split("/")
|
|
202
|
+
fw = parts[0] if len(parts) > 0 else ""
|
|
203
|
+
tk = parts[1] if len(parts) > 1 else ""
|
|
204
|
+
ft = parts[2] if len(parts) > 2 else ""
|
|
205
|
+
metrics.append(self.compute(runs, framework=fw, task=tk, failure_type=ft))
|
|
206
|
+
return metrics
|
|
207
|
+
|
|
208
|
+
def to_dataframe(self, metrics_list: list[AgentMetrics]) -> Any:
|
|
209
|
+
"""Convert a list of AgentMetrics to a pandas DataFrame."""
|
|
210
|
+
try:
|
|
211
|
+
import pandas as pd
|
|
212
|
+
except ImportError as e:
|
|
213
|
+
raise ImportError("pandas is required for to_dataframe(). pip install pandas") from e
|
|
214
|
+
rows = [m.to_dict() for m in metrics_list]
|
|
215
|
+
return pd.DataFrame(rows)
|
|
216
|
+
|
|
217
|
+
def _count_by_category(self, failures: list[Any]) -> dict[str, int]:
|
|
218
|
+
counts: dict[str, int] = {}
|
|
219
|
+
for r in failures:
|
|
220
|
+
cat = (
|
|
221
|
+
r.get("failure_category", "unknown")
|
|
222
|
+
if isinstance(r, dict)
|
|
223
|
+
else getattr(r, "failure_category", "unknown")
|
|
224
|
+
)
|
|
225
|
+
counts[cat] = counts.get(cat, 0) + 1
|
|
226
|
+
return counts
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# ---------------------------------------------------------------------------
|
|
230
|
+
# Helpers
|
|
231
|
+
# ---------------------------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _percentile(data: list[float], pct: int) -> float:
|
|
235
|
+
"""Ceiling nearest-rank percentile.
|
|
236
|
+
|
|
237
|
+
For P99 at n=30, ceil(30 * 99/100) = 30, so this returns the sample
|
|
238
|
+
maximum — consistent with the paper's convention that P99 equals the
|
|
239
|
+
empirical sample maximum for n < 100.
|
|
240
|
+
"""
|
|
241
|
+
if not data:
|
|
242
|
+
return 0.0
|
|
243
|
+
sorted_data = sorted(data)
|
|
244
|
+
import math
|
|
245
|
+
|
|
246
|
+
idx = min(math.ceil(len(sorted_data) * pct / 100), len(sorted_data)) - 1
|
|
247
|
+
return sorted_data[max(0, idx)]
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
"""Benchmark runner for AgentProbe.
|
|
2
|
+
|
|
3
|
+
Orchestrates multi-run experiments across agent frameworks, failure types,
|
|
4
|
+
and tasks. Produces structured RunResult records used by MetricsCalculator.
|
|
5
|
+
|
|
6
|
+
Usage::
|
|
7
|
+
|
|
8
|
+
from agentprobe.benchmark.runner import BenchmarkRunner, BenchmarkConfig
|
|
9
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
|
|
10
|
+
from agentprobe.agents.langgraph_agent import LangGraphToolAgent
|
|
11
|
+
|
|
12
|
+
config = BenchmarkConfig(
|
|
13
|
+
framework="langgraph",
|
|
14
|
+
task="tool_use",
|
|
15
|
+
n_runs=50,
|
|
16
|
+
failure_config=ToolFailureConfig(
|
|
17
|
+
failure_type=ToolFailureType.TIMEOUT,
|
|
18
|
+
failure_probability=0.2,
|
|
19
|
+
),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
agent = LangGraphToolAgent(llm=my_llm)
|
|
23
|
+
runner = BenchmarkRunner(agent=agent, config=config)
|
|
24
|
+
results = runner.run()
|
|
25
|
+
|
|
26
|
+
from agentprobe.benchmark.metrics import MetricsCalculator
|
|
27
|
+
metrics = MetricsCalculator().compute(results)
|
|
28
|
+
print(metrics)
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import concurrent.futures
|
|
34
|
+
import csv
|
|
35
|
+
import json
|
|
36
|
+
import time
|
|
37
|
+
from collections.abc import Callable
|
|
38
|
+
from dataclasses import dataclass, field
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import TYPE_CHECKING, Any
|
|
41
|
+
|
|
42
|
+
from opentelemetry import context as otel_context
|
|
43
|
+
from rich.console import Console
|
|
44
|
+
from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
|
|
45
|
+
|
|
46
|
+
from agentprobe.benchmark.metrics import _percentile
|
|
47
|
+
from agentprobe.observer.classifier import FailureClassifier
|
|
48
|
+
from agentprobe.observer.tracer import get_tracer
|
|
49
|
+
|
|
50
|
+
if TYPE_CHECKING:
|
|
51
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig
|
|
52
|
+
|
|
53
|
+
_tracer = get_tracer(__name__)
|
|
54
|
+
_console = Console()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _HarnessTimeout(Exception):
|
|
58
|
+
"""Raised when the benchmark runner's wall-clock deadline is exceeded.
|
|
59
|
+
|
|
60
|
+
Distinct from ``TimeoutError`` so that upstream timeouts (e.g. pyautogen's
|
|
61
|
+
``Completion.request_timeout``) are not misclassified as harness deadlines.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
# Data structures
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class RunResult:
|
|
72
|
+
"""Result of a single benchmark run (one agent invocation).
|
|
73
|
+
|
|
74
|
+
The ``status`` field distinguishes executed trials from those skipped
|
|
75
|
+
after a harness timeout aborted the condition:
|
|
76
|
+
- ``"completed"`` — trial was dispatched and finished (success or failure).
|
|
77
|
+
- ``"skipped"`` — trial was never dispatched (prior timeout lingering).
|
|
78
|
+
|
|
79
|
+
Only ``"completed"`` trials should be included in error-rate, recovery,
|
|
80
|
+
and latency calculations.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
run_id: int
|
|
84
|
+
framework: str
|
|
85
|
+
task: str
|
|
86
|
+
failure_type: str
|
|
87
|
+
success: bool
|
|
88
|
+
latency_ms: float
|
|
89
|
+
status: str = "completed"
|
|
90
|
+
failure_category: str = "none"
|
|
91
|
+
exception_type: str = ""
|
|
92
|
+
exception_msg: str = ""
|
|
93
|
+
recovered: bool = False
|
|
94
|
+
recovery_latency_ms: float = 0.0
|
|
95
|
+
recovery_output: str = ""
|
|
96
|
+
output: str = ""
|
|
97
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
98
|
+
|
|
99
|
+
def to_dict(self) -> dict[str, Any]:
|
|
100
|
+
return {
|
|
101
|
+
"run_id": self.run_id,
|
|
102
|
+
"framework": self.framework,
|
|
103
|
+
"task": self.task,
|
|
104
|
+
"failure_type": self.failure_type,
|
|
105
|
+
"status": self.status,
|
|
106
|
+
"success": self.success,
|
|
107
|
+
"latency_ms": round(self.latency_ms, 2),
|
|
108
|
+
"failure_category": self.failure_category,
|
|
109
|
+
"exception_type": self.exception_type,
|
|
110
|
+
"exception_msg": self.exception_msg,
|
|
111
|
+
"recovered": self.recovered,
|
|
112
|
+
"recovery_latency_ms": round(self.recovery_latency_ms, 2),
|
|
113
|
+
"recovery_output": self.recovery_output,
|
|
114
|
+
"output": self.output,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass
|
|
119
|
+
class BenchmarkConfig:
|
|
120
|
+
"""Configuration for one benchmark experiment.
|
|
121
|
+
|
|
122
|
+
Attributes:
|
|
123
|
+
framework: Agent framework identifier (e.g. "langgraph", "langchain").
|
|
124
|
+
task: Task identifier (e.g. "tool_use", "rag", "multi_agent").
|
|
125
|
+
n_runs: Number of independent runs per configuration.
|
|
126
|
+
failure_config: Tool failure injection configuration.
|
|
127
|
+
If None, runs without any injection (baseline).
|
|
128
|
+
recovery_attempts: How many times to retry after a failure.
|
|
129
|
+
Set to 0 to measure failure rate without recovery.
|
|
130
|
+
timeout_per_run_seconds: Wall-clock timeout per run.
|
|
131
|
+
seed: Global RNG seed for reproducibility.
|
|
132
|
+
save_results_path: If set, writes CSV/JSON results to this path.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
framework: str = "langgraph"
|
|
136
|
+
task: str = "tool_use"
|
|
137
|
+
n_runs: int = 30
|
|
138
|
+
failure_config: ToolFailureConfig | None = None
|
|
139
|
+
recovery_attempts: int = 1
|
|
140
|
+
timeout_per_run_seconds: float = 60.0
|
|
141
|
+
seed: int = 42
|
|
142
|
+
save_results_path: str | None = None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ---------------------------------------------------------------------------
|
|
146
|
+
# Runner
|
|
147
|
+
# ---------------------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class BenchmarkRunner:
|
|
151
|
+
"""Runs benchmark experiments for one (framework, task, failure_config) triple.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
agent: Any object satisfying :class:`agentprobe.agents.BaseAgent`
|
|
155
|
+
(``invoke(input: dict) -> dict`` plus ``reset()``), e.g. the
|
|
156
|
+
LangGraph, LangChain, or AutoGen adapters in ``agentprobe.agents``.
|
|
157
|
+
config: Benchmark configuration.
|
|
158
|
+
task_fn: Optional callable that generates task inputs per run.
|
|
159
|
+
Signature: ``(run_id: int, task: str) -> dict``.
|
|
160
|
+
Defaults to :func:`paper_task_fn`, the five-prompt set used in the
|
|
161
|
+
paper. Use ``agentprobe.benchmark.tasks.corpus_task_fn(task)`` to
|
|
162
|
+
run against the larger graded corpora instead.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
def __init__(
|
|
166
|
+
self,
|
|
167
|
+
agent: Any,
|
|
168
|
+
config: BenchmarkConfig | None = None,
|
|
169
|
+
task_fn: Callable[[int, str], dict] | None = None,
|
|
170
|
+
) -> None:
|
|
171
|
+
self.agent = agent
|
|
172
|
+
self.config = config or BenchmarkConfig()
|
|
173
|
+
self._task_fn = task_fn or paper_task_fn
|
|
174
|
+
self._classifier: Any = FailureClassifier()
|
|
175
|
+
self._invocation_hung = False
|
|
176
|
+
|
|
177
|
+
def run(self) -> list[RunResult]:
|
|
178
|
+
"""Execute the full benchmark. Returns a list of RunResult objects."""
|
|
179
|
+
cfg = self.config
|
|
180
|
+
failure_type = cfg.failure_config.failure_type.value if cfg.failure_config else "none"
|
|
181
|
+
|
|
182
|
+
_console.rule(
|
|
183
|
+
f"[bold blue]AgentProbe Benchmark — {cfg.framework}/{cfg.task}/{failure_type}"
|
|
184
|
+
)
|
|
185
|
+
_console.print(f" Runs: {cfg.n_runs} | Recovery attempts: {cfg.recovery_attempts}")
|
|
186
|
+
|
|
187
|
+
results: list[RunResult] = []
|
|
188
|
+
|
|
189
|
+
with Progress(
|
|
190
|
+
SpinnerColumn(),
|
|
191
|
+
"[progress.description]{task.description}",
|
|
192
|
+
TimeElapsedColumn(),
|
|
193
|
+
console=_console,
|
|
194
|
+
) as progress:
|
|
195
|
+
task_bar = progress.add_task(f"[cyan]Running {cfg.n_runs} trials...", total=cfg.n_runs)
|
|
196
|
+
|
|
197
|
+
for run_id in range(cfg.n_runs):
|
|
198
|
+
if self._invocation_hung:
|
|
199
|
+
results.append(
|
|
200
|
+
RunResult(
|
|
201
|
+
run_id=run_id,
|
|
202
|
+
framework=cfg.framework,
|
|
203
|
+
task=cfg.task,
|
|
204
|
+
failure_type=failure_type,
|
|
205
|
+
success=False,
|
|
206
|
+
latency_ms=0.0,
|
|
207
|
+
status="skipped",
|
|
208
|
+
failure_category="harness_abort",
|
|
209
|
+
exception_type="_HarnessTimeout",
|
|
210
|
+
exception_msg="Skipped: prior trial timed out with lingering thread",
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
progress.advance(task_bar)
|
|
214
|
+
continue
|
|
215
|
+
result = self._run_single(run_id, failure_type)
|
|
216
|
+
results.append(result)
|
|
217
|
+
progress.advance(task_bar)
|
|
218
|
+
|
|
219
|
+
_console.print(self._summary_line(results))
|
|
220
|
+
|
|
221
|
+
if cfg.save_results_path:
|
|
222
|
+
self._save(results, cfg.save_results_path)
|
|
223
|
+
|
|
224
|
+
return results
|
|
225
|
+
|
|
226
|
+
# ------------------------------------------------------------------
|
|
227
|
+
# Internal
|
|
228
|
+
# ------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
def _run_single(self, run_id: int, failure_type: str) -> RunResult:
|
|
231
|
+
cfg = self.config
|
|
232
|
+
task_input = self._task_fn(run_id, cfg.task)
|
|
233
|
+
|
|
234
|
+
with _tracer.start_as_current_span(
|
|
235
|
+
f"benchmark.run.{run_id}",
|
|
236
|
+
attributes={
|
|
237
|
+
"run.id": run_id,
|
|
238
|
+
"framework": cfg.framework,
|
|
239
|
+
"task": cfg.task,
|
|
240
|
+
"failure_type": failure_type,
|
|
241
|
+
},
|
|
242
|
+
):
|
|
243
|
+
start = time.perf_counter()
|
|
244
|
+
try:
|
|
245
|
+
output = self._invoke_with_timeout(task_input)
|
|
246
|
+
latency_ms = (time.perf_counter() - start) * 1000
|
|
247
|
+
return RunResult(
|
|
248
|
+
run_id=run_id,
|
|
249
|
+
framework=cfg.framework,
|
|
250
|
+
task=cfg.task,
|
|
251
|
+
failure_type=failure_type,
|
|
252
|
+
success=True,
|
|
253
|
+
latency_ms=latency_ms,
|
|
254
|
+
output=str(output),
|
|
255
|
+
)
|
|
256
|
+
except Exception as exc:
|
|
257
|
+
latency_ms = (time.perf_counter() - start) * 1000
|
|
258
|
+
category = "unknown"
|
|
259
|
+
if self._classifier is not None:
|
|
260
|
+
event = self._classifier.classify_exception(exc, framework=cfg.framework)
|
|
261
|
+
category = event.category.value
|
|
262
|
+
|
|
263
|
+
# Attempt recovery
|
|
264
|
+
recovered, recovery_latency_ms, recovery_output = self._attempt_recovery(task_input)
|
|
265
|
+
|
|
266
|
+
return RunResult(
|
|
267
|
+
run_id=run_id,
|
|
268
|
+
framework=cfg.framework,
|
|
269
|
+
task=cfg.task,
|
|
270
|
+
failure_type=failure_type,
|
|
271
|
+
success=False,
|
|
272
|
+
latency_ms=latency_ms,
|
|
273
|
+
failure_category=category,
|
|
274
|
+
exception_type=type(exc).__name__,
|
|
275
|
+
exception_msg=str(exc)[:200],
|
|
276
|
+
recovered=recovered,
|
|
277
|
+
recovery_latency_ms=recovery_latency_ms,
|
|
278
|
+
recovery_output=str(recovery_output) if recovery_output else "",
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
def _invoke_with_timeout(self, task_input: dict) -> Any:
|
|
282
|
+
"""Invoke agent with wall-clock timeout enforcement.
|
|
283
|
+
|
|
284
|
+
Uses a thread so that the agent's injector RNG, injection logs,
|
|
285
|
+
and all mutable state advance in-process (no fork/copy issues).
|
|
286
|
+
The current OpenTelemetry context is explicitly propagated into the
|
|
287
|
+
worker thread so that agent spans become children of the benchmark span.
|
|
288
|
+
|
|
289
|
+
Guarantees:
|
|
290
|
+
1. Upstream exceptions (including ``TimeoutError``) are propagated
|
|
291
|
+
with their original type and message.
|
|
292
|
+
2. Only genuine wall-clock deadline expiry raises ``_HarnessTimeout``.
|
|
293
|
+
3. After a harness timeout, ``self._invocation_hung`` is set to
|
|
294
|
+
``True`` so that ``_attempt_recovery`` skips retry and
|
|
295
|
+
``run()`` aborts remaining trials, preventing overlap between
|
|
296
|
+
the lingering thread and new invocations.
|
|
297
|
+
|
|
298
|
+
Limitation: Python threads cannot be forcibly killed. A hung
|
|
299
|
+
thread may linger until the process exits.
|
|
300
|
+
"""
|
|
301
|
+
timeout = self.config.timeout_per_run_seconds
|
|
302
|
+
ctx = otel_context.get_current()
|
|
303
|
+
|
|
304
|
+
def _worker() -> Any:
|
|
305
|
+
token = otel_context.attach(ctx)
|
|
306
|
+
try:
|
|
307
|
+
return self.agent.invoke(task_input)
|
|
308
|
+
finally:
|
|
309
|
+
otel_context.detach(token)
|
|
310
|
+
|
|
311
|
+
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
|
312
|
+
future = pool.submit(_worker)
|
|
313
|
+
pool.shutdown(wait=False)
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
return future.result(timeout=timeout)
|
|
317
|
+
except Exception:
|
|
318
|
+
if future.done():
|
|
319
|
+
# Worker finished (possibly with an exception) before
|
|
320
|
+
# the deadline — propagate the original exception as-is.
|
|
321
|
+
return future.result() # re-raises the worker exception
|
|
322
|
+
# Worker still running — genuine deadline miss.
|
|
323
|
+
self._invocation_hung = True
|
|
324
|
+
raise _HarnessTimeout(f"Agent invocation exceeded {timeout}s wall-clock timeout")
|
|
325
|
+
|
|
326
|
+
def _attempt_recovery(self, task_input: dict) -> tuple[bool, float, Any]:
|
|
327
|
+
"""Try to invoke the agent again after a failure.
|
|
328
|
+
|
|
329
|
+
If the prior invocation is still running (hung thread after a
|
|
330
|
+
harness timeout), recovery is skipped entirely to prevent two
|
|
331
|
+
concurrent invocations on the same agent.
|
|
332
|
+
|
|
333
|
+
Returns (recovered, latency_ms, output).
|
|
334
|
+
"""
|
|
335
|
+
if self.config.recovery_attempts == 0:
|
|
336
|
+
return False, 0.0, None
|
|
337
|
+
if self._invocation_hung:
|
|
338
|
+
return False, 0.0, None
|
|
339
|
+
for _ in range(self.config.recovery_attempts):
|
|
340
|
+
start = time.perf_counter()
|
|
341
|
+
try:
|
|
342
|
+
output = self._invoke_with_timeout(task_input)
|
|
343
|
+
return True, (time.perf_counter() - start) * 1000, output
|
|
344
|
+
except Exception:
|
|
345
|
+
if self._invocation_hung:
|
|
346
|
+
break # recovery attempt itself timed out; stop retrying
|
|
347
|
+
continue
|
|
348
|
+
return False, 0.0, None
|
|
349
|
+
|
|
350
|
+
def _summary_line(self, results: list[RunResult]) -> str:
|
|
351
|
+
executed = [r for r in results if r.status == "completed"]
|
|
352
|
+
skipped = len(results) - len(executed)
|
|
353
|
+
total = len(executed)
|
|
354
|
+
failures = sum(1 for r in executed if not r.success)
|
|
355
|
+
rate = failures / total if total > 0 else 0.0
|
|
356
|
+
recovered = sum(1 for r in executed if r.recovered)
|
|
357
|
+
latencies = [r.latency_ms for r in executed]
|
|
358
|
+
p99 = _percentile(latencies, 99)
|
|
359
|
+
line = (
|
|
360
|
+
f"[green]Done[/green] — {total} executed | "
|
|
361
|
+
f"failure_rate={rate:.1%} | "
|
|
362
|
+
f"recovered={recovered}/{failures} | "
|
|
363
|
+
f"P99={p99:.0f}ms"
|
|
364
|
+
)
|
|
365
|
+
if skipped:
|
|
366
|
+
line += f" | [yellow]{skipped} skipped (condition incomplete)[/yellow]"
|
|
367
|
+
return line
|
|
368
|
+
|
|
369
|
+
def _save(self, results: list[RunResult], path: str) -> None:
|
|
370
|
+
p = Path(path)
|
|
371
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
372
|
+
|
|
373
|
+
# CSV
|
|
374
|
+
csv_path = p.with_suffix(".csv")
|
|
375
|
+
with open(csv_path, "w", newline="") as f:
|
|
376
|
+
if results:
|
|
377
|
+
writer = csv.DictWriter(f, fieldnames=results[0].to_dict().keys())
|
|
378
|
+
writer.writeheader()
|
|
379
|
+
writer.writerows(r.to_dict() for r in results)
|
|
380
|
+
|
|
381
|
+
# JSON
|
|
382
|
+
json_path = p.with_suffix(".json")
|
|
383
|
+
with open(json_path, "w") as f:
|
|
384
|
+
json.dump([r.to_dict() for r in results], f, indent=2)
|
|
385
|
+
|
|
386
|
+
_console.print(f"[dim]Results saved to {csv_path} and {json_path}[/dim]")
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# ---------------------------------------------------------------------------
|
|
390
|
+
# Default task generator — the paper's prompt set
|
|
391
|
+
# ---------------------------------------------------------------------------
|
|
392
|
+
|
|
393
|
+
#: The five prompts per task type used in every experiment reported in the paper
|
|
394
|
+
#: (Phase 1, Phase 2, and cross-model validation). Run ``i`` receives prompt
|
|
395
|
+
#: ``i % 5``. Kept verbatim so that published batches reproduce exactly.
|
|
396
|
+
#: The larger graded corpora in :mod:`agentprobe.benchmark.tasks` are *not* the
|
|
397
|
+
#: paper's prompts; use ``corpus_task_fn()`` from that module to benchmark
|
|
398
|
+
#: against them instead.
|
|
399
|
+
PAPER_PROMPTS: dict[str, list[str]] = {
|
|
400
|
+
"tool_use": [
|
|
401
|
+
"What is the capital of France?",
|
|
402
|
+
"Search for recent news about LLM reliability.",
|
|
403
|
+
"Calculate 15% of 847.",
|
|
404
|
+
"What is the current weather in San Francisco?",
|
|
405
|
+
"Find the top 3 papers on multi-agent systems published in 2024.",
|
|
406
|
+
],
|
|
407
|
+
"rag": [
|
|
408
|
+
"Summarize the main contributions of the AgentProbe paper.",
|
|
409
|
+
"What are the 5 failure categories in the AgentProbe taxonomy?",
|
|
410
|
+
"How does AgentProbe measure MTTR?",
|
|
411
|
+
"Compare LangGraph and LangChain reliability profiles.",
|
|
412
|
+
"What observability gaps exist in current agentic frameworks?",
|
|
413
|
+
],
|
|
414
|
+
"multi_agent": [
|
|
415
|
+
"Research and then summarize the latest advances in LLM agents.",
|
|
416
|
+
"Find relevant papers and write a literature review outline.",
|
|
417
|
+
"Analyze the dataset and produce a visualization plan.",
|
|
418
|
+
"Draft a blog post about AI reliability best practices.",
|
|
419
|
+
"Create a benchmark plan for testing agent robustness.",
|
|
420
|
+
],
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def paper_task_fn(run_id: int, task: str) -> dict[str, Any]:
|
|
425
|
+
"""Default ``task_fn``: cycle through :data:`PAPER_PROMPTS` for *task*.
|
|
426
|
+
|
|
427
|
+
Unknown task names fall back to the ``tool_use`` prompts.
|
|
428
|
+
"""
|
|
429
|
+
inputs = PAPER_PROMPTS.get(task, PAPER_PROMPTS["tool_use"])
|
|
430
|
+
return {"input": inputs[run_id % len(inputs)], "run_id": run_id}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
# Backwards-compatible alias used by the experiment scripts.
|
|
434
|
+
_default_task_fn = paper_task_fn
|