agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
"""Statistical analysis module for AgentProbe benchmark results.
|
|
2
|
+
|
|
3
|
+
Provides confidence intervals, effect sizes, hypothesis tests, and
|
|
4
|
+
summary reports for comparing agent framework reliability metrics.
|
|
5
|
+
|
|
6
|
+
Uses only ``scipy.stats`` and ``numpy`` -- no exotic dependencies.
|
|
7
|
+
|
|
8
|
+
Usage::
|
|
9
|
+
|
|
10
|
+
from agentprobe.benchmark.statistics import (
|
|
11
|
+
proportion_ci,
|
|
12
|
+
mean_ci,
|
|
13
|
+
cohens_h,
|
|
14
|
+
cohens_d,
|
|
15
|
+
compare_proportions,
|
|
16
|
+
compare_distributions,
|
|
17
|
+
compare_multiple_groups,
|
|
18
|
+
build_comparison_report,
|
|
19
|
+
StatisticalReport,
|
|
20
|
+
)
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import math
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from typing import Literal
|
|
28
|
+
|
|
29
|
+
import numpy as np
|
|
30
|
+
from scipy import stats
|
|
31
|
+
|
|
32
|
+
# ---------------------------------------------------------------------------
|
|
33
|
+
# Confidence Intervals
|
|
34
|
+
# ---------------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def proportion_ci(
|
|
38
|
+
successes: int,
|
|
39
|
+
total: int,
|
|
40
|
+
confidence: float = 0.95,
|
|
41
|
+
) -> tuple[float, float]:
|
|
42
|
+
"""Wilson score interval for a proportion.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
successes:
|
|
47
|
+
Number of successful trials.
|
|
48
|
+
total:
|
|
49
|
+
Total number of trials.
|
|
50
|
+
confidence:
|
|
51
|
+
Confidence level (default 0.95 for a 95% CI).
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
(lower, upper) bounds of the confidence interval.
|
|
56
|
+
|
|
57
|
+
Raises
|
|
58
|
+
------
|
|
59
|
+
ValueError
|
|
60
|
+
If *total* is zero or *successes* is outside [0, total].
|
|
61
|
+
"""
|
|
62
|
+
if total <= 0:
|
|
63
|
+
raise ValueError("total must be > 0")
|
|
64
|
+
if successes < 0 or successes > total:
|
|
65
|
+
raise ValueError("successes must be in [0, total]")
|
|
66
|
+
|
|
67
|
+
z = stats.norm.ppf(1 - (1 - confidence) / 2)
|
|
68
|
+
p_hat = successes / total
|
|
69
|
+
denominator = 1 + z**2 / total
|
|
70
|
+
centre = (p_hat + z**2 / (2 * total)) / denominator
|
|
71
|
+
margin = (z / denominator) * math.sqrt(p_hat * (1 - p_hat) / total + z**2 / (4 * total**2))
|
|
72
|
+
return (max(0.0, centre - margin), min(1.0, centre + margin))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def mean_ci(
|
|
76
|
+
values: list[float] | np.ndarray,
|
|
77
|
+
confidence: float = 0.95,
|
|
78
|
+
) -> tuple[float, float]:
|
|
79
|
+
"""Confidence interval for the mean using the t-distribution.
|
|
80
|
+
|
|
81
|
+
Parameters
|
|
82
|
+
----------
|
|
83
|
+
values:
|
|
84
|
+
Sample observations.
|
|
85
|
+
confidence:
|
|
86
|
+
Confidence level.
|
|
87
|
+
|
|
88
|
+
Returns
|
|
89
|
+
-------
|
|
90
|
+
(lower, upper) bounds.
|
|
91
|
+
|
|
92
|
+
Raises
|
|
93
|
+
------
|
|
94
|
+
ValueError
|
|
95
|
+
If *values* has fewer than 2 elements.
|
|
96
|
+
"""
|
|
97
|
+
arr = np.asarray(values, dtype=np.float64)
|
|
98
|
+
n = len(arr)
|
|
99
|
+
if n < 2:
|
|
100
|
+
raise ValueError("Need at least 2 observations for a CI")
|
|
101
|
+
mean = float(np.mean(arr))
|
|
102
|
+
se = float(stats.sem(arr))
|
|
103
|
+
alpha = 1 - confidence
|
|
104
|
+
t_crit = float(stats.t.ppf(1 - alpha / 2, df=n - 1))
|
|
105
|
+
return (mean - t_crit * se, mean + t_crit * se)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
# Effect Sizes
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
EffectMagnitude = Literal["negligible", "small", "medium", "large"]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _interpret_d(d: float) -> EffectMagnitude:
|
|
116
|
+
"""Interpret Cohen's d magnitude per Cohen (1988)."""
|
|
117
|
+
ad = abs(d)
|
|
118
|
+
if ad < 0.2:
|
|
119
|
+
return "negligible"
|
|
120
|
+
if ad < 0.5:
|
|
121
|
+
return "small"
|
|
122
|
+
if ad < 0.8:
|
|
123
|
+
return "medium"
|
|
124
|
+
return "large"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _interpret_h(h: float) -> EffectMagnitude:
|
|
128
|
+
"""Interpret Cohen's h magnitude per Cohen (1988)."""
|
|
129
|
+
ah = abs(h)
|
|
130
|
+
if ah < 0.2:
|
|
131
|
+
return "negligible"
|
|
132
|
+
if ah < 0.5:
|
|
133
|
+
return "small"
|
|
134
|
+
if ah < 0.8:
|
|
135
|
+
return "medium"
|
|
136
|
+
return "large"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cohens_h(p1: float, p2: float) -> tuple[float, EffectMagnitude]:
|
|
140
|
+
"""Cohen's *h* for comparing two proportions.
|
|
141
|
+
|
|
142
|
+
Parameters
|
|
143
|
+
----------
|
|
144
|
+
p1, p2:
|
|
145
|
+
Proportions in [0, 1].
|
|
146
|
+
|
|
147
|
+
Returns
|
|
148
|
+
-------
|
|
149
|
+
(h_value, interpretation) where interpretation is one of
|
|
150
|
+
``"negligible"``, ``"small"``, ``"medium"``, ``"large"``.
|
|
151
|
+
"""
|
|
152
|
+
h = 2 * (math.asin(math.sqrt(p1)) - math.asin(math.sqrt(p2)))
|
|
153
|
+
return (h, _interpret_h(h))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def cohens_d(
|
|
157
|
+
group1: list[float] | np.ndarray,
|
|
158
|
+
group2: list[float] | np.ndarray,
|
|
159
|
+
) -> tuple[float, EffectMagnitude]:
|
|
160
|
+
"""Cohen's *d* for comparing two continuous samples (pooled SD).
|
|
161
|
+
|
|
162
|
+
Parameters
|
|
163
|
+
----------
|
|
164
|
+
group1, group2:
|
|
165
|
+
Sample observations.
|
|
166
|
+
|
|
167
|
+
Returns
|
|
168
|
+
-------
|
|
169
|
+
(d_value, interpretation).
|
|
170
|
+
"""
|
|
171
|
+
a1 = np.asarray(group1, dtype=np.float64)
|
|
172
|
+
a2 = np.asarray(group2, dtype=np.float64)
|
|
173
|
+
n1, n2 = len(a1), len(a2)
|
|
174
|
+
if n1 < 2 or n2 < 2:
|
|
175
|
+
raise ValueError("Both groups need at least 2 observations")
|
|
176
|
+
var1, var2 = float(np.var(a1, ddof=1)), float(np.var(a2, ddof=1))
|
|
177
|
+
pooled_std = math.sqrt(((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2))
|
|
178
|
+
if pooled_std == 0:
|
|
179
|
+
d = 0.0
|
|
180
|
+
else:
|
|
181
|
+
d = (float(np.mean(a1)) - float(np.mean(a2))) / pooled_std
|
|
182
|
+
return (d, _interpret_d(d))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------------------
|
|
186
|
+
# Hypothesis Tests
|
|
187
|
+
# ---------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass
|
|
191
|
+
class HypothesisTestResult:
|
|
192
|
+
"""Outcome of a statistical hypothesis test."""
|
|
193
|
+
|
|
194
|
+
test_name: str
|
|
195
|
+
statistic: float
|
|
196
|
+
p_value: float
|
|
197
|
+
significant: bool # at alpha used
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def compare_proportions(
|
|
201
|
+
successes1: int,
|
|
202
|
+
total1: int,
|
|
203
|
+
successes2: int,
|
|
204
|
+
total2: int,
|
|
205
|
+
alpha: float = 0.05,
|
|
206
|
+
) -> HypothesisTestResult:
|
|
207
|
+
"""Compare two proportions using Fisher's exact test (for small samples)
|
|
208
|
+
or Chi-squared test (otherwise).
|
|
209
|
+
|
|
210
|
+
Chooses Fisher's exact when any expected cell count < 5, otherwise
|
|
211
|
+
uses chi-squared with Yates' continuity correction.
|
|
212
|
+
|
|
213
|
+
Parameters
|
|
214
|
+
----------
|
|
215
|
+
successes1, total1:
|
|
216
|
+
Successes and total trials for group 1.
|
|
217
|
+
successes2, total2:
|
|
218
|
+
Successes and total trials for group 2.
|
|
219
|
+
alpha:
|
|
220
|
+
Significance level (default 0.05).
|
|
221
|
+
"""
|
|
222
|
+
failures1 = total1 - successes1
|
|
223
|
+
failures2 = total2 - successes2
|
|
224
|
+
table = np.array([[successes1, failures1], [successes2, failures2]])
|
|
225
|
+
|
|
226
|
+
# Expected cell counts
|
|
227
|
+
row_sums = table.sum(axis=1)
|
|
228
|
+
col_sums = table.sum(axis=0)
|
|
229
|
+
n = table.sum()
|
|
230
|
+
expected = np.outer(row_sums, col_sums) / n if n > 0 else table
|
|
231
|
+
|
|
232
|
+
if np.any(expected < 5):
|
|
233
|
+
# Fisher's exact test
|
|
234
|
+
odds_ratio, p = stats.fisher_exact(table)
|
|
235
|
+
return HypothesisTestResult(
|
|
236
|
+
test_name="Fisher's exact test",
|
|
237
|
+
statistic=odds_ratio,
|
|
238
|
+
p_value=float(p),
|
|
239
|
+
significant=float(p) < alpha,
|
|
240
|
+
)
|
|
241
|
+
else:
|
|
242
|
+
chi2, p, _, _ = stats.chi2_contingency(table, correction=True)
|
|
243
|
+
return HypothesisTestResult(
|
|
244
|
+
test_name="Chi-squared test (Yates)",
|
|
245
|
+
statistic=float(chi2),
|
|
246
|
+
p_value=float(p),
|
|
247
|
+
significant=float(p) < alpha,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def compare_distributions(
|
|
252
|
+
group1: list[float] | np.ndarray,
|
|
253
|
+
group2: list[float] | np.ndarray,
|
|
254
|
+
alpha: float = 0.05,
|
|
255
|
+
) -> HypothesisTestResult:
|
|
256
|
+
"""Mann-Whitney U test for comparing two continuous distributions.
|
|
257
|
+
|
|
258
|
+
Non-parametric -- does not assume normality.
|
|
259
|
+
"""
|
|
260
|
+
a1 = np.asarray(group1, dtype=np.float64)
|
|
261
|
+
a2 = np.asarray(group2, dtype=np.float64)
|
|
262
|
+
if len(a1) < 1 or len(a2) < 1:
|
|
263
|
+
raise ValueError("Both groups need at least 1 observation")
|
|
264
|
+
stat, p = stats.mannwhitneyu(a1, a2, alternative="two-sided")
|
|
265
|
+
return HypothesisTestResult(
|
|
266
|
+
test_name="Mann-Whitney U test",
|
|
267
|
+
statistic=float(stat),
|
|
268
|
+
p_value=float(p),
|
|
269
|
+
significant=float(p) < alpha,
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def compare_multiple_groups(
|
|
274
|
+
groups: list[list[float] | np.ndarray],
|
|
275
|
+
alpha: float = 0.05,
|
|
276
|
+
) -> HypothesisTestResult:
|
|
277
|
+
"""Kruskal-Wallis H-test for comparing 3+ independent groups.
|
|
278
|
+
|
|
279
|
+
Non-parametric -- suitable for latency comparisons across frameworks.
|
|
280
|
+
"""
|
|
281
|
+
if len(groups) < 2:
|
|
282
|
+
raise ValueError("Need at least 2 groups")
|
|
283
|
+
arrays = [np.asarray(g, dtype=np.float64) for g in groups]
|
|
284
|
+
stat, p = stats.kruskal(*arrays)
|
|
285
|
+
return HypothesisTestResult(
|
|
286
|
+
test_name="Kruskal-Wallis H test",
|
|
287
|
+
statistic=float(stat),
|
|
288
|
+
p_value=float(p),
|
|
289
|
+
significant=float(p) < alpha,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# ---------------------------------------------------------------------------
|
|
294
|
+
# Summary Report
|
|
295
|
+
# ---------------------------------------------------------------------------
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass
|
|
299
|
+
class PairwiseComparison:
|
|
300
|
+
"""Statistical comparison between two frameworks."""
|
|
301
|
+
|
|
302
|
+
framework_a: str
|
|
303
|
+
framework_b: str
|
|
304
|
+
|
|
305
|
+
# Proportion (failure rate) comparison
|
|
306
|
+
failure_rate_a: float
|
|
307
|
+
failure_rate_b: float
|
|
308
|
+
failure_rate_ci_a: tuple[float, float]
|
|
309
|
+
failure_rate_ci_b: tuple[float, float]
|
|
310
|
+
failure_rate_test: HypothesisTestResult
|
|
311
|
+
failure_rate_effect: tuple[float, EffectMagnitude]
|
|
312
|
+
|
|
313
|
+
# Latency comparison
|
|
314
|
+
latency_test: HypothesisTestResult | None = None
|
|
315
|
+
latency_effect: tuple[float, EffectMagnitude] | None = None
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
@dataclass
|
|
319
|
+
class StatisticalReport:
|
|
320
|
+
"""Full statistical comparison report across frameworks.
|
|
321
|
+
|
|
322
|
+
Attributes
|
|
323
|
+
----------
|
|
324
|
+
pairwise:
|
|
325
|
+
List of PairwiseComparison for each framework pair.
|
|
326
|
+
omnibus_latency:
|
|
327
|
+
Kruskal-Wallis test across all frameworks (if 3+).
|
|
328
|
+
n_frameworks:
|
|
329
|
+
Number of frameworks compared.
|
|
330
|
+
alpha:
|
|
331
|
+
Significance level used for all tests.
|
|
332
|
+
"""
|
|
333
|
+
|
|
334
|
+
pairwise: list[PairwiseComparison] = field(default_factory=list)
|
|
335
|
+
omnibus_latency: HypothesisTestResult | None = None
|
|
336
|
+
n_frameworks: int = 0
|
|
337
|
+
alpha: float = 0.05
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
@dataclass
|
|
341
|
+
class FrameworkData:
|
|
342
|
+
"""Input data for a single framework to build_comparison_report.
|
|
343
|
+
|
|
344
|
+
Attributes
|
|
345
|
+
----------
|
|
346
|
+
name:
|
|
347
|
+
Framework identifier (e.g. ``"langgraph"``).
|
|
348
|
+
successes:
|
|
349
|
+
Number of successful runs.
|
|
350
|
+
total:
|
|
351
|
+
Total runs.
|
|
352
|
+
latencies:
|
|
353
|
+
Per-run latency values in milliseconds.
|
|
354
|
+
"""
|
|
355
|
+
|
|
356
|
+
name: str
|
|
357
|
+
successes: int
|
|
358
|
+
total: int
|
|
359
|
+
latencies: list[float] | np.ndarray
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def build_comparison_report(
|
|
363
|
+
frameworks: list[FrameworkData],
|
|
364
|
+
alpha: float = 0.05,
|
|
365
|
+
confidence: float = 0.95,
|
|
366
|
+
) -> StatisticalReport:
|
|
367
|
+
"""Produce a full pairwise statistical comparison across frameworks.
|
|
368
|
+
|
|
369
|
+
Parameters
|
|
370
|
+
----------
|
|
371
|
+
frameworks:
|
|
372
|
+
One FrameworkData per framework.
|
|
373
|
+
alpha:
|
|
374
|
+
Significance level for hypothesis tests.
|
|
375
|
+
confidence:
|
|
376
|
+
Confidence level for CIs.
|
|
377
|
+
|
|
378
|
+
Returns
|
|
379
|
+
-------
|
|
380
|
+
A StatisticalReport with pairwise comparisons and an omnibus
|
|
381
|
+
latency test (when 3+ frameworks are provided).
|
|
382
|
+
"""
|
|
383
|
+
report = StatisticalReport(n_frameworks=len(frameworks), alpha=alpha)
|
|
384
|
+
|
|
385
|
+
# Pairwise comparisons
|
|
386
|
+
for i in range(len(frameworks)):
|
|
387
|
+
for j in range(i + 1, len(frameworks)):
|
|
388
|
+
fa, fb = frameworks[i], frameworks[j]
|
|
389
|
+
|
|
390
|
+
rate_a = (fa.total - fa.successes) / fa.total if fa.total else 0.0
|
|
391
|
+
rate_b = (fb.total - fb.successes) / fb.total if fb.total else 0.0
|
|
392
|
+
ci_a = proportion_ci(fa.total - fa.successes, fa.total, confidence)
|
|
393
|
+
ci_b = proportion_ci(fb.total - fb.successes, fb.total, confidence)
|
|
394
|
+
|
|
395
|
+
prop_test = compare_proportions(
|
|
396
|
+
fa.total - fa.successes,
|
|
397
|
+
fa.total,
|
|
398
|
+
fb.total - fb.successes,
|
|
399
|
+
fb.total,
|
|
400
|
+
alpha=alpha,
|
|
401
|
+
)
|
|
402
|
+
h_val, h_interp = cohens_h(rate_a, rate_b)
|
|
403
|
+
|
|
404
|
+
lat_arr_a = np.asarray(fa.latencies, dtype=np.float64)
|
|
405
|
+
lat_arr_b = np.asarray(fb.latencies, dtype=np.float64)
|
|
406
|
+
|
|
407
|
+
lat_test = None
|
|
408
|
+
lat_effect = None
|
|
409
|
+
if len(lat_arr_a) >= 2 and len(lat_arr_b) >= 2:
|
|
410
|
+
lat_test = compare_distributions(lat_arr_a, lat_arr_b, alpha=alpha)
|
|
411
|
+
lat_effect = cohens_d(lat_arr_a, lat_arr_b)
|
|
412
|
+
|
|
413
|
+
report.pairwise.append(
|
|
414
|
+
PairwiseComparison(
|
|
415
|
+
framework_a=fa.name,
|
|
416
|
+
framework_b=fb.name,
|
|
417
|
+
failure_rate_a=rate_a,
|
|
418
|
+
failure_rate_b=rate_b,
|
|
419
|
+
failure_rate_ci_a=ci_a,
|
|
420
|
+
failure_rate_ci_b=ci_b,
|
|
421
|
+
failure_rate_test=prop_test,
|
|
422
|
+
failure_rate_effect=(h_val, h_interp),
|
|
423
|
+
latency_test=lat_test,
|
|
424
|
+
latency_effect=lat_effect,
|
|
425
|
+
)
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
# Omnibus latency test (3+ frameworks)
|
|
429
|
+
if len(frameworks) >= 3:
|
|
430
|
+
lat_groups = [
|
|
431
|
+
np.asarray(f.latencies, dtype=np.float64)
|
|
432
|
+
for f in frameworks
|
|
433
|
+
if len(np.asarray(f.latencies)) >= 2
|
|
434
|
+
]
|
|
435
|
+
if len(lat_groups) >= 3:
|
|
436
|
+
report.omnibus_latency = compare_multiple_groups(lat_groups, alpha=alpha)
|
|
437
|
+
|
|
438
|
+
return report
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
# ---------------------------------------------------------------------------
|
|
442
|
+
# Public API
|
|
443
|
+
# ---------------------------------------------------------------------------
|
|
444
|
+
|
|
445
|
+
__all__ = [
|
|
446
|
+
"proportion_ci",
|
|
447
|
+
"mean_ci",
|
|
448
|
+
"cohens_h",
|
|
449
|
+
"cohens_d",
|
|
450
|
+
"compare_proportions",
|
|
451
|
+
"compare_distributions",
|
|
452
|
+
"compare_multiple_groups",
|
|
453
|
+
"build_comparison_report",
|
|
454
|
+
"EffectMagnitude",
|
|
455
|
+
"HypothesisTestResult",
|
|
456
|
+
"PairwiseComparison",
|
|
457
|
+
"StatisticalReport",
|
|
458
|
+
"FrameworkData",
|
|
459
|
+
]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Standard benchmark task corpora for AgentProbe.
|
|
2
|
+
|
|
3
|
+
Tasks are designed to be:
|
|
4
|
+
1. Reproducible — deterministic inputs, public data sources
|
|
5
|
+
2. Comparable — same inputs applied across all frameworks
|
|
6
|
+
3. Realistic — representative of production agentic use cases
|
|
7
|
+
|
|
8
|
+
Task categories:
|
|
9
|
+
- tool_use : Single-agent, multi-tool calling (25 queries)
|
|
10
|
+
- rag : Retrieval-augmented generation over a static corpus (15 queries)
|
|
11
|
+
- multi_agent : Two-agent researcher + writer pipeline (15 queries)
|
|
12
|
+
|
|
13
|
+
**Relationship to the paper.** The experiments reported in the paper did *not*
|
|
14
|
+
use these corpora. They used the five-prompt-per-task set in
|
|
15
|
+
:data:`agentprobe.benchmark.runner.PAPER_PROMPTS`, which is also the
|
|
16
|
+
:class:`~agentprobe.benchmark.runner.BenchmarkRunner` default so that the
|
|
17
|
+
published batches reproduce exactly. To benchmark against the larger corpora
|
|
18
|
+
instead, pass :func:`corpus_task_fn` as the runner's ``task_fn``::
|
|
19
|
+
|
|
20
|
+
from agentprobe.benchmark import BenchmarkRunner, BenchmarkConfig, corpus_task_fn
|
|
21
|
+
|
|
22
|
+
runner = BenchmarkRunner(
|
|
23
|
+
agent=agent,
|
|
24
|
+
config=BenchmarkConfig(task="tool_use", n_runs=50),
|
|
25
|
+
task_fn=corpus_task_fn("tool_use"),
|
|
26
|
+
)
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from collections.abc import Callable
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from agentprobe.benchmark.tasks.multi_agent import MultiAgentTask
|
|
35
|
+
from agentprobe.benchmark.tasks.rag import RAGTask
|
|
36
|
+
from agentprobe.benchmark.tasks.tool_use import ToolUseTask
|
|
37
|
+
|
|
38
|
+
__all__ = ["ToolUseTask", "RAGTask", "MultiAgentTask", "get_task", "corpus_task_fn"]
|
|
39
|
+
|
|
40
|
+
_TASK_CLASSES: dict[str, type] = {
|
|
41
|
+
ToolUseTask.name: ToolUseTask,
|
|
42
|
+
RAGTask.name: RAGTask,
|
|
43
|
+
MultiAgentTask.name: MultiAgentTask,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_task(name: str) -> ToolUseTask | RAGTask | MultiAgentTask:
|
|
48
|
+
"""Return a task instance by name (``"tool_use"``, ``"rag"``, ``"multi_agent"``)."""
|
|
49
|
+
try:
|
|
50
|
+
cls = _TASK_CLASSES[name]
|
|
51
|
+
except KeyError:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"Unknown task {name!r}; expected one of {sorted(_TASK_CLASSES)}"
|
|
54
|
+
) from None
|
|
55
|
+
return cls() # type: ignore[no-any-return]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def corpus_task_fn(name: str) -> Callable[[int, str], dict[str, Any]]:
|
|
59
|
+
"""Build a ``task_fn`` for :class:`~agentprobe.benchmark.runner.BenchmarkRunner`
|
|
60
|
+
that cycles through the full query corpus for *name*.
|
|
61
|
+
|
|
62
|
+
Run ``i`` receives query ``i % len(corpus)``. The returned dict carries the
|
|
63
|
+
``query_id`` so outputs can be graded with ``get_task(name).grade(...)``.
|
|
64
|
+
"""
|
|
65
|
+
task = get_task(name)
|
|
66
|
+
inputs = task.get_inputs()
|
|
67
|
+
if not inputs:
|
|
68
|
+
raise ValueError(f"Task {name!r} has no queries")
|
|
69
|
+
|
|
70
|
+
def _task_fn(run_id: int, _task_name: str) -> dict[str, Any]:
|
|
71
|
+
item = dict(inputs[run_id % len(inputs)])
|
|
72
|
+
item["run_id"] = run_id
|
|
73
|
+
return item
|
|
74
|
+
|
|
75
|
+
return _task_fn
|