agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,459 @@
1
+ """Statistical analysis module for AgentProbe benchmark results.
2
+
3
+ Provides confidence intervals, effect sizes, hypothesis tests, and
4
+ summary reports for comparing agent framework reliability metrics.
5
+
6
+ Uses only ``scipy.stats`` and ``numpy`` -- no exotic dependencies.
7
+
8
+ Usage::
9
+
10
+ from agentprobe.benchmark.statistics import (
11
+ proportion_ci,
12
+ mean_ci,
13
+ cohens_h,
14
+ cohens_d,
15
+ compare_proportions,
16
+ compare_distributions,
17
+ compare_multiple_groups,
18
+ build_comparison_report,
19
+ StatisticalReport,
20
+ )
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import math
26
+ from dataclasses import dataclass, field
27
+ from typing import Literal
28
+
29
+ import numpy as np
30
+ from scipy import stats
31
+
32
+ # ---------------------------------------------------------------------------
33
+ # Confidence Intervals
34
+ # ---------------------------------------------------------------------------
35
+
36
+
37
+ def proportion_ci(
38
+ successes: int,
39
+ total: int,
40
+ confidence: float = 0.95,
41
+ ) -> tuple[float, float]:
42
+ """Wilson score interval for a proportion.
43
+
44
+ Parameters
45
+ ----------
46
+ successes:
47
+ Number of successful trials.
48
+ total:
49
+ Total number of trials.
50
+ confidence:
51
+ Confidence level (default 0.95 for a 95% CI).
52
+
53
+ Returns
54
+ -------
55
+ (lower, upper) bounds of the confidence interval.
56
+
57
+ Raises
58
+ ------
59
+ ValueError
60
+ If *total* is zero or *successes* is outside [0, total].
61
+ """
62
+ if total <= 0:
63
+ raise ValueError("total must be > 0")
64
+ if successes < 0 or successes > total:
65
+ raise ValueError("successes must be in [0, total]")
66
+
67
+ z = stats.norm.ppf(1 - (1 - confidence) / 2)
68
+ p_hat = successes / total
69
+ denominator = 1 + z**2 / total
70
+ centre = (p_hat + z**2 / (2 * total)) / denominator
71
+ margin = (z / denominator) * math.sqrt(p_hat * (1 - p_hat) / total + z**2 / (4 * total**2))
72
+ return (max(0.0, centre - margin), min(1.0, centre + margin))
73
+
74
+
75
+ def mean_ci(
76
+ values: list[float] | np.ndarray,
77
+ confidence: float = 0.95,
78
+ ) -> tuple[float, float]:
79
+ """Confidence interval for the mean using the t-distribution.
80
+
81
+ Parameters
82
+ ----------
83
+ values:
84
+ Sample observations.
85
+ confidence:
86
+ Confidence level.
87
+
88
+ Returns
89
+ -------
90
+ (lower, upper) bounds.
91
+
92
+ Raises
93
+ ------
94
+ ValueError
95
+ If *values* has fewer than 2 elements.
96
+ """
97
+ arr = np.asarray(values, dtype=np.float64)
98
+ n = len(arr)
99
+ if n < 2:
100
+ raise ValueError("Need at least 2 observations for a CI")
101
+ mean = float(np.mean(arr))
102
+ se = float(stats.sem(arr))
103
+ alpha = 1 - confidence
104
+ t_crit = float(stats.t.ppf(1 - alpha / 2, df=n - 1))
105
+ return (mean - t_crit * se, mean + t_crit * se)
106
+
107
+
108
+ # ---------------------------------------------------------------------------
109
+ # Effect Sizes
110
+ # ---------------------------------------------------------------------------
111
+
112
+ EffectMagnitude = Literal["negligible", "small", "medium", "large"]
113
+
114
+
115
+ def _interpret_d(d: float) -> EffectMagnitude:
116
+ """Interpret Cohen's d magnitude per Cohen (1988)."""
117
+ ad = abs(d)
118
+ if ad < 0.2:
119
+ return "negligible"
120
+ if ad < 0.5:
121
+ return "small"
122
+ if ad < 0.8:
123
+ return "medium"
124
+ return "large"
125
+
126
+
127
+ def _interpret_h(h: float) -> EffectMagnitude:
128
+ """Interpret Cohen's h magnitude per Cohen (1988)."""
129
+ ah = abs(h)
130
+ if ah < 0.2:
131
+ return "negligible"
132
+ if ah < 0.5:
133
+ return "small"
134
+ if ah < 0.8:
135
+ return "medium"
136
+ return "large"
137
+
138
+
139
+ def cohens_h(p1: float, p2: float) -> tuple[float, EffectMagnitude]:
140
+ """Cohen's *h* for comparing two proportions.
141
+
142
+ Parameters
143
+ ----------
144
+ p1, p2:
145
+ Proportions in [0, 1].
146
+
147
+ Returns
148
+ -------
149
+ (h_value, interpretation) where interpretation is one of
150
+ ``"negligible"``, ``"small"``, ``"medium"``, ``"large"``.
151
+ """
152
+ h = 2 * (math.asin(math.sqrt(p1)) - math.asin(math.sqrt(p2)))
153
+ return (h, _interpret_h(h))
154
+
155
+
156
+ def cohens_d(
157
+ group1: list[float] | np.ndarray,
158
+ group2: list[float] | np.ndarray,
159
+ ) -> tuple[float, EffectMagnitude]:
160
+ """Cohen's *d* for comparing two continuous samples (pooled SD).
161
+
162
+ Parameters
163
+ ----------
164
+ group1, group2:
165
+ Sample observations.
166
+
167
+ Returns
168
+ -------
169
+ (d_value, interpretation).
170
+ """
171
+ a1 = np.asarray(group1, dtype=np.float64)
172
+ a2 = np.asarray(group2, dtype=np.float64)
173
+ n1, n2 = len(a1), len(a2)
174
+ if n1 < 2 or n2 < 2:
175
+ raise ValueError("Both groups need at least 2 observations")
176
+ var1, var2 = float(np.var(a1, ddof=1)), float(np.var(a2, ddof=1))
177
+ pooled_std = math.sqrt(((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2))
178
+ if pooled_std == 0:
179
+ d = 0.0
180
+ else:
181
+ d = (float(np.mean(a1)) - float(np.mean(a2))) / pooled_std
182
+ return (d, _interpret_d(d))
183
+
184
+
185
+ # ---------------------------------------------------------------------------
186
+ # Hypothesis Tests
187
+ # ---------------------------------------------------------------------------
188
+
189
+
190
+ @dataclass
191
+ class HypothesisTestResult:
192
+ """Outcome of a statistical hypothesis test."""
193
+
194
+ test_name: str
195
+ statistic: float
196
+ p_value: float
197
+ significant: bool # at alpha used
198
+
199
+
200
+ def compare_proportions(
201
+ successes1: int,
202
+ total1: int,
203
+ successes2: int,
204
+ total2: int,
205
+ alpha: float = 0.05,
206
+ ) -> HypothesisTestResult:
207
+ """Compare two proportions using Fisher's exact test (for small samples)
208
+ or Chi-squared test (otherwise).
209
+
210
+ Chooses Fisher's exact when any expected cell count < 5, otherwise
211
+ uses chi-squared with Yates' continuity correction.
212
+
213
+ Parameters
214
+ ----------
215
+ successes1, total1:
216
+ Successes and total trials for group 1.
217
+ successes2, total2:
218
+ Successes and total trials for group 2.
219
+ alpha:
220
+ Significance level (default 0.05).
221
+ """
222
+ failures1 = total1 - successes1
223
+ failures2 = total2 - successes2
224
+ table = np.array([[successes1, failures1], [successes2, failures2]])
225
+
226
+ # Expected cell counts
227
+ row_sums = table.sum(axis=1)
228
+ col_sums = table.sum(axis=0)
229
+ n = table.sum()
230
+ expected = np.outer(row_sums, col_sums) / n if n > 0 else table
231
+
232
+ if np.any(expected < 5):
233
+ # Fisher's exact test
234
+ odds_ratio, p = stats.fisher_exact(table)
235
+ return HypothesisTestResult(
236
+ test_name="Fisher's exact test",
237
+ statistic=odds_ratio,
238
+ p_value=float(p),
239
+ significant=float(p) < alpha,
240
+ )
241
+ else:
242
+ chi2, p, _, _ = stats.chi2_contingency(table, correction=True)
243
+ return HypothesisTestResult(
244
+ test_name="Chi-squared test (Yates)",
245
+ statistic=float(chi2),
246
+ p_value=float(p),
247
+ significant=float(p) < alpha,
248
+ )
249
+
250
+
251
+ def compare_distributions(
252
+ group1: list[float] | np.ndarray,
253
+ group2: list[float] | np.ndarray,
254
+ alpha: float = 0.05,
255
+ ) -> HypothesisTestResult:
256
+ """Mann-Whitney U test for comparing two continuous distributions.
257
+
258
+ Non-parametric -- does not assume normality.
259
+ """
260
+ a1 = np.asarray(group1, dtype=np.float64)
261
+ a2 = np.asarray(group2, dtype=np.float64)
262
+ if len(a1) < 1 or len(a2) < 1:
263
+ raise ValueError("Both groups need at least 1 observation")
264
+ stat, p = stats.mannwhitneyu(a1, a2, alternative="two-sided")
265
+ return HypothesisTestResult(
266
+ test_name="Mann-Whitney U test",
267
+ statistic=float(stat),
268
+ p_value=float(p),
269
+ significant=float(p) < alpha,
270
+ )
271
+
272
+
273
+ def compare_multiple_groups(
274
+ groups: list[list[float] | np.ndarray],
275
+ alpha: float = 0.05,
276
+ ) -> HypothesisTestResult:
277
+ """Kruskal-Wallis H-test for comparing 3+ independent groups.
278
+
279
+ Non-parametric -- suitable for latency comparisons across frameworks.
280
+ """
281
+ if len(groups) < 2:
282
+ raise ValueError("Need at least 2 groups")
283
+ arrays = [np.asarray(g, dtype=np.float64) for g in groups]
284
+ stat, p = stats.kruskal(*arrays)
285
+ return HypothesisTestResult(
286
+ test_name="Kruskal-Wallis H test",
287
+ statistic=float(stat),
288
+ p_value=float(p),
289
+ significant=float(p) < alpha,
290
+ )
291
+
292
+
293
+ # ---------------------------------------------------------------------------
294
+ # Summary Report
295
+ # ---------------------------------------------------------------------------
296
+
297
+
298
+ @dataclass
299
+ class PairwiseComparison:
300
+ """Statistical comparison between two frameworks."""
301
+
302
+ framework_a: str
303
+ framework_b: str
304
+
305
+ # Proportion (failure rate) comparison
306
+ failure_rate_a: float
307
+ failure_rate_b: float
308
+ failure_rate_ci_a: tuple[float, float]
309
+ failure_rate_ci_b: tuple[float, float]
310
+ failure_rate_test: HypothesisTestResult
311
+ failure_rate_effect: tuple[float, EffectMagnitude]
312
+
313
+ # Latency comparison
314
+ latency_test: HypothesisTestResult | None = None
315
+ latency_effect: tuple[float, EffectMagnitude] | None = None
316
+
317
+
318
+ @dataclass
319
+ class StatisticalReport:
320
+ """Full statistical comparison report across frameworks.
321
+
322
+ Attributes
323
+ ----------
324
+ pairwise:
325
+ List of PairwiseComparison for each framework pair.
326
+ omnibus_latency:
327
+ Kruskal-Wallis test across all frameworks (if 3+).
328
+ n_frameworks:
329
+ Number of frameworks compared.
330
+ alpha:
331
+ Significance level used for all tests.
332
+ """
333
+
334
+ pairwise: list[PairwiseComparison] = field(default_factory=list)
335
+ omnibus_latency: HypothesisTestResult | None = None
336
+ n_frameworks: int = 0
337
+ alpha: float = 0.05
338
+
339
+
340
+ @dataclass
341
+ class FrameworkData:
342
+ """Input data for a single framework to build_comparison_report.
343
+
344
+ Attributes
345
+ ----------
346
+ name:
347
+ Framework identifier (e.g. ``"langgraph"``).
348
+ successes:
349
+ Number of successful runs.
350
+ total:
351
+ Total runs.
352
+ latencies:
353
+ Per-run latency values in milliseconds.
354
+ """
355
+
356
+ name: str
357
+ successes: int
358
+ total: int
359
+ latencies: list[float] | np.ndarray
360
+
361
+
362
+ def build_comparison_report(
363
+ frameworks: list[FrameworkData],
364
+ alpha: float = 0.05,
365
+ confidence: float = 0.95,
366
+ ) -> StatisticalReport:
367
+ """Produce a full pairwise statistical comparison across frameworks.
368
+
369
+ Parameters
370
+ ----------
371
+ frameworks:
372
+ One FrameworkData per framework.
373
+ alpha:
374
+ Significance level for hypothesis tests.
375
+ confidence:
376
+ Confidence level for CIs.
377
+
378
+ Returns
379
+ -------
380
+ A StatisticalReport with pairwise comparisons and an omnibus
381
+ latency test (when 3+ frameworks are provided).
382
+ """
383
+ report = StatisticalReport(n_frameworks=len(frameworks), alpha=alpha)
384
+
385
+ # Pairwise comparisons
386
+ for i in range(len(frameworks)):
387
+ for j in range(i + 1, len(frameworks)):
388
+ fa, fb = frameworks[i], frameworks[j]
389
+
390
+ rate_a = (fa.total - fa.successes) / fa.total if fa.total else 0.0
391
+ rate_b = (fb.total - fb.successes) / fb.total if fb.total else 0.0
392
+ ci_a = proportion_ci(fa.total - fa.successes, fa.total, confidence)
393
+ ci_b = proportion_ci(fb.total - fb.successes, fb.total, confidence)
394
+
395
+ prop_test = compare_proportions(
396
+ fa.total - fa.successes,
397
+ fa.total,
398
+ fb.total - fb.successes,
399
+ fb.total,
400
+ alpha=alpha,
401
+ )
402
+ h_val, h_interp = cohens_h(rate_a, rate_b)
403
+
404
+ lat_arr_a = np.asarray(fa.latencies, dtype=np.float64)
405
+ lat_arr_b = np.asarray(fb.latencies, dtype=np.float64)
406
+
407
+ lat_test = None
408
+ lat_effect = None
409
+ if len(lat_arr_a) >= 2 and len(lat_arr_b) >= 2:
410
+ lat_test = compare_distributions(lat_arr_a, lat_arr_b, alpha=alpha)
411
+ lat_effect = cohens_d(lat_arr_a, lat_arr_b)
412
+
413
+ report.pairwise.append(
414
+ PairwiseComparison(
415
+ framework_a=fa.name,
416
+ framework_b=fb.name,
417
+ failure_rate_a=rate_a,
418
+ failure_rate_b=rate_b,
419
+ failure_rate_ci_a=ci_a,
420
+ failure_rate_ci_b=ci_b,
421
+ failure_rate_test=prop_test,
422
+ failure_rate_effect=(h_val, h_interp),
423
+ latency_test=lat_test,
424
+ latency_effect=lat_effect,
425
+ )
426
+ )
427
+
428
+ # Omnibus latency test (3+ frameworks)
429
+ if len(frameworks) >= 3:
430
+ lat_groups = [
431
+ np.asarray(f.latencies, dtype=np.float64)
432
+ for f in frameworks
433
+ if len(np.asarray(f.latencies)) >= 2
434
+ ]
435
+ if len(lat_groups) >= 3:
436
+ report.omnibus_latency = compare_multiple_groups(lat_groups, alpha=alpha)
437
+
438
+ return report
439
+
440
+
441
+ # ---------------------------------------------------------------------------
442
+ # Public API
443
+ # ---------------------------------------------------------------------------
444
+
445
+ __all__ = [
446
+ "proportion_ci",
447
+ "mean_ci",
448
+ "cohens_h",
449
+ "cohens_d",
450
+ "compare_proportions",
451
+ "compare_distributions",
452
+ "compare_multiple_groups",
453
+ "build_comparison_report",
454
+ "EffectMagnitude",
455
+ "HypothesisTestResult",
456
+ "PairwiseComparison",
457
+ "StatisticalReport",
458
+ "FrameworkData",
459
+ ]
@@ -0,0 +1,75 @@
1
+ """Standard benchmark task corpora for AgentProbe.
2
+
3
+ Tasks are designed to be:
4
+ 1. Reproducible — deterministic inputs, public data sources
5
+ 2. Comparable — same inputs applied across all frameworks
6
+ 3. Realistic — representative of production agentic use cases
7
+
8
+ Task categories:
9
+ - tool_use : Single-agent, multi-tool calling (25 queries)
10
+ - rag : Retrieval-augmented generation over a static corpus (15 queries)
11
+ - multi_agent : Two-agent researcher + writer pipeline (15 queries)
12
+
13
+ **Relationship to the paper.** The experiments reported in the paper did *not*
14
+ use these corpora. They used the five-prompt-per-task set in
15
+ :data:`agentprobe.benchmark.runner.PAPER_PROMPTS`, which is also the
16
+ :class:`~agentprobe.benchmark.runner.BenchmarkRunner` default so that the
17
+ published batches reproduce exactly. To benchmark against the larger corpora
18
+ instead, pass :func:`corpus_task_fn` as the runner's ``task_fn``::
19
+
20
+ from agentprobe.benchmark import BenchmarkRunner, BenchmarkConfig, corpus_task_fn
21
+
22
+ runner = BenchmarkRunner(
23
+ agent=agent,
24
+ config=BenchmarkConfig(task="tool_use", n_runs=50),
25
+ task_fn=corpus_task_fn("tool_use"),
26
+ )
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ from collections.abc import Callable
32
+ from typing import Any
33
+
34
+ from agentprobe.benchmark.tasks.multi_agent import MultiAgentTask
35
+ from agentprobe.benchmark.tasks.rag import RAGTask
36
+ from agentprobe.benchmark.tasks.tool_use import ToolUseTask
37
+
38
+ __all__ = ["ToolUseTask", "RAGTask", "MultiAgentTask", "get_task", "corpus_task_fn"]
39
+
40
+ _TASK_CLASSES: dict[str, type] = {
41
+ ToolUseTask.name: ToolUseTask,
42
+ RAGTask.name: RAGTask,
43
+ MultiAgentTask.name: MultiAgentTask,
44
+ }
45
+
46
+
47
+ def get_task(name: str) -> ToolUseTask | RAGTask | MultiAgentTask:
48
+ """Return a task instance by name (``"tool_use"``, ``"rag"``, ``"multi_agent"``)."""
49
+ try:
50
+ cls = _TASK_CLASSES[name]
51
+ except KeyError:
52
+ raise ValueError(
53
+ f"Unknown task {name!r}; expected one of {sorted(_TASK_CLASSES)}"
54
+ ) from None
55
+ return cls() # type: ignore[no-any-return]
56
+
57
+
58
+ def corpus_task_fn(name: str) -> Callable[[int, str], dict[str, Any]]:
59
+ """Build a ``task_fn`` for :class:`~agentprobe.benchmark.runner.BenchmarkRunner`
60
+ that cycles through the full query corpus for *name*.
61
+
62
+ Run ``i`` receives query ``i % len(corpus)``. The returned dict carries the
63
+ ``query_id`` so outputs can be graded with ``get_task(name).grade(...)``.
64
+ """
65
+ task = get_task(name)
66
+ inputs = task.get_inputs()
67
+ if not inputs:
68
+ raise ValueError(f"Task {name!r} has no queries")
69
+
70
+ def _task_fn(run_id: int, _task_name: str) -> dict[str, Any]:
71
+ item = dict(inputs[run_id % len(inputs)])
72
+ item["run_id"] = run_id
73
+ return item
74
+
75
+ return _task_fn