agentprobe-evals 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentprobe/ci.py ADDED
@@ -0,0 +1,216 @@
1
+ """One-shot CI gate: run a baseline and a candidate agent, compare, report, and set an exit code.
2
+
3
+ Exit codes:
4
+ 0 no regression (or --no-fail)
5
+ 1 statistically significant regression
6
+ 2 infrastructure problem (a run failed or too many tasks errored), so there is no verdict.
7
+ This is deliberately separate from 1: an outage in the agent's endpoint must not be
8
+ reported as "the change made the agent worse", and must not be silently treated as a pass.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import html
14
+ import json
15
+ import os
16
+ from dataclasses import dataclass, field
17
+
18
+ from sqlalchemy import select
19
+
20
+ from .compare import Comparison, compare_runs
21
+ from .db import Database, Run, Task, TaskResult
22
+ from .telemetry import tracer
23
+
24
+ MAX_ERROR_RATE = 0.2
25
+ OK_STATUSES = ("completed", "budget_exceeded")
26
+
27
+
28
+ @dataclass
29
+ class CIOutcome:
30
+ verdict: str # regression | improvement | no_significant_change | infrastructure_error
31
+ comparison: Comparison | None = None
32
+ problems: list[str] = field(default_factory=list)
33
+ broken: list[dict] = field(default_factory=list)
34
+ fixed: list[dict] = field(default_factory=list)
35
+ budget_limited: bool = False
36
+ alpha: float = 0.05
37
+
38
+ @property
39
+ def exit_code_failure(self) -> int:
40
+ return {"regression": 1, "infrastructure_error": 2}.get(self.verdict, 0)
41
+
42
+
43
+ def _flips(s, baseline: Run, candidate: Run) -> tuple[list[dict], list[dict]]:
44
+ tasks = {t.id: t for t in s.scalars(select(Task).where(Task.dataset_id == baseline.dataset_id))}
45
+ base = {r.task_id: r for r in baseline.results}
46
+ cand = {r.task_id: r for r in candidate.results}
47
+
48
+ def row(tid: int) -> dict:
49
+ return {
50
+ "input": tasks[tid].input,
51
+ "baseline": base[tid].output,
52
+ "candidate": cand[tid].output,
53
+ "reasoning": cand[tid].judge_reasoning,
54
+ }
55
+
56
+ common = sorted(base.keys() & cand.keys())
57
+ broken = [row(t) for t in common if base[t].passed and not cand[t].passed]
58
+ fixed = [row(t) for t in common if not base[t].passed and cand[t].passed]
59
+ return broken, fixed
60
+
61
+
62
+ def _health(run: Run, results: list[TaskResult], name: str) -> list[str]:
63
+ problems = []
64
+ if run.status not in OK_STATUSES:
65
+ problems.append(f"{name} run ended as '{run.status}'" + (f": {run.error}" if run.error else ""))
66
+ if not results:
67
+ problems.append(f"{name} run produced no results")
68
+ else:
69
+ errors = sum(1 for r in results if r.error)
70
+ if errors / len(results) > MAX_ERROR_RATE:
71
+ sample = next(r.error for r in results if r.error)
72
+ problems.append(
73
+ f"{name}: {errors}/{len(results)} tasks errored (limit {MAX_ERROR_RATE:.0%}); e.g. {sample[:140]}"
74
+ )
75
+ return problems
76
+
77
+
78
+ def run_ci(
79
+ db: Database,
80
+ dataset_path: str,
81
+ baseline_agent: str,
82
+ candidate_agent: str,
83
+ concurrency: int = 4,
84
+ budget_usd: float | None = None,
85
+ alpha: float = 0.05,
86
+ ) -> CIOutcome:
87
+ from .cli import _load_dataset # same loader as `agentprobe run`
88
+ from .runner import execute_run
89
+
90
+ dataset_id = _load_dataset(db, dataset_path)
91
+ ids = {}
92
+ # One trace for the whole gate: ci -> (baseline run, candidate run) -> tasks -> agent/judge calls.
93
+ with tracer().start_as_current_span("agentprobe.ci", attributes={"agentprobe.alpha": alpha}):
94
+ for label, agent in (("baseline", baseline_agent), ("candidate", candidate_agent)):
95
+ with db.session() as s:
96
+ run = Run(
97
+ dataset_id=dataset_id, agent=agent, label=label, concurrency=concurrency, budget_usd=budget_usd
98
+ )
99
+ s.add(run)
100
+ s.commit()
101
+ ids[label] = run.id
102
+ execute_run(db, ids[label]) # sequential: the two runs must not compete for one endpoint
103
+
104
+ with db.session() as s:
105
+ b, c = s.get(Run, ids["baseline"]), s.get(Run, ids["candidate"])
106
+ problems = _health(b, list(b.results), "baseline") + _health(c, list(c.results), "candidate")
107
+ if problems:
108
+ return CIOutcome("infrastructure_error", problems=problems, alpha=alpha)
109
+ cmp = compare_runs(list(b.results), list(c.results), alpha)
110
+ broken, fixed = _flips(s, b, c)
111
+ limited = "budget_exceeded" in (b.status, c.status)
112
+ return CIOutcome(cmp.verdict, cmp, [], broken, fixed, limited, alpha)
113
+
114
+
115
+ # ---------- reporting ----------
116
+
117
+ def _cell(text: str, limit: int = 110) -> str:
118
+ """Make untrusted text safe for a markdown table cell (agent output can contain anything)."""
119
+ flat = " ".join(str(text).split())
120
+ if len(flat) > limit:
121
+ flat = flat[: limit - 1] + "…"
122
+ return html.escape(flat).replace("|", "\\|").replace("`", "`")
123
+
124
+
125
+ def _flip_table(rows: list[dict], title: str, max_rows: int = 10) -> str:
126
+ if not rows:
127
+ return ""
128
+ body = "\n".join(
129
+ f"| {_cell(r['input'], 80)} | {_cell(r['baseline'])} | {_cell(r['candidate'])} |" for r in rows[:max_rows]
130
+ )
131
+ more = f"\n\n…and {len(rows) - max_rows} more." if len(rows) > max_rows else ""
132
+ return (
133
+ f"<details><summary>{title} ({len(rows)})</summary>\n\n"
134
+ f"| Task | Baseline output | Candidate output |\n|---|---|---|\n{body}{more}\n\n</details>\n"
135
+ )
136
+
137
+
138
+ def _sensitivity_line(c: Comparison) -> str:
139
+ """Say how much this dataset can actually detect, so 'no significant change' is read correctly."""
140
+ from .power import min_detectable_effect
141
+
142
+ if c.common_tasks == 0:
143
+ return ""
144
+ observed = (c.broken + c.fixed) / c.common_tasks
145
+ discordance = min(1.0, max(observed, 0.05)) # floor: a quiet run must not claim infinite power
146
+ mde = min_detectable_effect(c.common_tasks, discordance)
147
+ basis = f"{c.common_tasks} tasks, about {discordance:.0%} of them flipping between versions"
148
+ if mde is None:
149
+ return f"<sub>Sensitivity: with {basis}, this comparison cannot reach 80% power, so a non-significant result says little.</sub>"
150
+ return f"<sub>Sensitivity: with {basis}, changes of {mde * 100:.0f} points or more are detected with 80% power.</sub>"
151
+
152
+
153
+ def render_markdown(o: CIOutcome, baseline_agent: str = "", candidate_agent: str = "") -> str:
154
+ if o.verdict == "infrastructure_error":
155
+ items = "\n".join(f"- {_cell(p, 300)}" for p in o.problems)
156
+ return (
157
+ "## AgentProbe: ⚠️ No verdict (infrastructure problem)\n\n"
158
+ "The comparison could not be made, so this is **not** a statement about the change.\n\n"
159
+ f"{items}\n"
160
+ )
161
+ c = o.comparison
162
+ assert c is not None
163
+ head = {
164
+ "regression": "❌ Regression detected",
165
+ "improvement": "✅ Improvement",
166
+ "no_significant_change": "✅ No significant change",
167
+ }[o.verdict]
168
+ delta = c.pass_rate_delta * 100
169
+ cost_b, cost_c = c.baseline.mean_cost_usd, c.candidate.mean_cost_usd
170
+ cost_row = (
171
+ f"| Mean cost / task | ${cost_b:.6f} | ${cost_c:.6f} | |\n" if (cost_b or cost_c) else ""
172
+ )
173
+ sig = "significant" if c.significant else "not significant"
174
+ out = (
175
+ f"## AgentProbe: {head}\n\n"
176
+ f"| | Baseline | Candidate | Δ |\n|---|---|---|---|\n"
177
+ f"| Pass rate | {c.baseline.pass_rate:.1%} | {c.candidate.pass_rate:.1%} | {delta:+.1f} pts |\n"
178
+ f"{cost_row}"
179
+ f"| p95 latency | {c.baseline.p95_latency_ms:.0f} ms | {c.candidate.p95_latency_ms:.0f} ms | |\n"
180
+ f"| Errored tasks | {c.baseline.errors} | {c.candidate.errors} | |\n\n"
181
+ f"**{c.broken} broke · {c.fixed} fixed** across {c.common_tasks} shared tasks. "
182
+ f"Exact McNemar p = {c.p_value:.4f} ({sig} at α = {o.alpha}).\n\n"
183
+ )
184
+ out += _sensitivity_line(c) + "\n\n"
185
+ if baseline_agent or candidate_agent:
186
+ out += f"<sub>baseline `{_cell(baseline_agent, 80)}` · candidate `{_cell(candidate_agent, 80)}`</sub>\n\n"
187
+ if o.budget_limited:
188
+ out += "> ⚠️ A run hit its cost budget and stopped early; only tasks both runs finished are compared.\n\n"
189
+ out += _flip_table(o.broken, "Tasks the candidate broke") + _flip_table(o.fixed, "Tasks the candidate fixed")
190
+ return out
191
+
192
+
193
+ def write_github_outputs(o: CIOutcome, markdown: str) -> None:
194
+ """Publish the job summary and step outputs when running under GitHub Actions."""
195
+ if path := os.environ.get("GITHUB_STEP_SUMMARY"):
196
+ with open(path, "a") as fh:
197
+ fh.write(markdown + "\n")
198
+ if path := os.environ.get("GITHUB_OUTPUT"):
199
+ c = o.comparison
200
+ values = {
201
+ "verdict": o.verdict,
202
+ "baseline-pass-rate": f"{c.baseline.pass_rate:.4f}" if c else "",
203
+ "candidate-pass-rate": f"{c.candidate.pass_rate:.4f}" if c else "",
204
+ "p-value": f"{c.p_value:.6f}" if c else "",
205
+ "broken": str(c.broken) if c else "",
206
+ "fixed": str(c.fixed) if c else "",
207
+ }
208
+ with open(path, "a") as fh:
209
+ fh.writelines(f"{key}={value}\n" for key, value in values.items())
210
+
211
+
212
+ def to_json(o: CIOutcome) -> str:
213
+ return json.dumps(
214
+ {"verdict": o.verdict, "problems": o.problems, "comparison": o.comparison.to_dict() if o.comparison else None},
215
+ indent=2,
216
+ )
agentprobe/cli.py ADDED
@@ -0,0 +1,326 @@
1
+ """Command line interface, designed so CI can gate merges on agent quality.
2
+
3
+ agentprobe run --dataset examples/dataset.json --agent mock:accuracy=0.9 --label v1
4
+ agentprobe compare --baseline 1 --candidate 2 --fail-on-regression
5
+ agentprobe ci --dataset d.json --baseline-agent A --candidate-agent B # both steps, for CI
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import os
13
+ import sys
14
+
15
+ from sqlalchemy import select
16
+
17
+ from .compare import compare_runs
18
+ from .db import Database, Dataset, Run, Task
19
+ from .runner import execute_run
20
+
21
+
22
+ def _load_dataset(db: Database, path: str) -> int:
23
+ with open(path) as fh:
24
+ data = json.load(fh)
25
+ with db.session() as s:
26
+ ds = s.scalar(select(Dataset).where(Dataset.name == data["name"]))
27
+ if ds is None:
28
+ ds = Dataset(name=data["name"], tasks=[Task(**t) for t in data["tasks"]])
29
+ s.add(ds)
30
+ s.commit()
31
+ return ds.id
32
+
33
+
34
+ def cmd_run(args: argparse.Namespace) -> int:
35
+ db = Database(args.db)
36
+ dataset_id = _load_dataset(db, args.dataset)
37
+ with db.session() as s:
38
+ run = Run(
39
+ dataset_id=dataset_id,
40
+ agent=args.agent,
41
+ label=args.label,
42
+ concurrency=args.concurrency,
43
+ budget_usd=args.budget,
44
+ )
45
+ s.add(run)
46
+ s.commit()
47
+ run_id = run.id
48
+ execute_run(db, run_id)
49
+ with db.session() as s:
50
+ run = s.get(Run, run_id)
51
+ passed = sum(r.passed for r in run.results)
52
+ print(f"run {run_id} [{run.status}] {passed}/{len(run.results)} passed ({args.agent})")
53
+ return 0 if run.status in ("completed", "budget_exceeded") else 1
54
+
55
+
56
+ def cmd_compare(args: argparse.Namespace) -> int:
57
+ db = Database(args.db)
58
+ with db.session() as s:
59
+ b, c = s.get(Run, args.baseline), s.get(Run, args.candidate)
60
+ if b is None or c is None:
61
+ print("run not found", file=sys.stderr)
62
+ return 2
63
+ cmp = compare_runs(list(b.results), list(c.results), args.alpha)
64
+ print(json.dumps(cmp.to_dict(), indent=2))
65
+ print(
66
+ f"\n{cmp.verdict.upper()}: pass rate {cmp.baseline.pass_rate:.1%} -> "
67
+ f"{cmp.candidate.pass_rate:.1%} (broken={cmp.broken}, fixed={cmp.fixed}, p={cmp.p_value:.4f})"
68
+ )
69
+ return 1 if args.fail_on_regression and cmp.verdict == "regression" else 0
70
+
71
+
72
+ def cmd_calibrate(args: argparse.Namespace) -> int:
73
+ from .calibration import run_calibration
74
+ from .db import HumanLabel
75
+
76
+ db = Database(args.db)
77
+ with db.session() as s:
78
+ if s.get(Run, args.run) is None:
79
+ print("run not found", file=sys.stderr)
80
+ return 2
81
+ if args.labels:
82
+ with open(args.labels) as fh:
83
+ labels = json.load(fh) # [{"task_id": 1, "passed": true}, ...]
84
+ existing = {
85
+ lb.task_id: lb for lb in s.scalars(select(HumanLabel).where(HumanLabel.run_id == args.run))
86
+ }
87
+ for item in labels:
88
+ if item["task_id"] in existing:
89
+ existing[item["task_id"]].passed = bool(item["passed"])
90
+ else:
91
+ s.add(HumanLabel(run_id=args.run, task_id=item["task_id"], passed=bool(item["passed"])))
92
+ s.commit()
93
+ cal = run_calibration(s, args.run, args.min_n)
94
+ print(json.dumps(cal.to_dict(), indent=2))
95
+ print(f"\n{cal.reliability.upper()}: kappa={cal.kappa:.2f}, agreement={cal.accuracy:.1%} on {cal.n} labels. {cal.note}")
96
+ return 1 if args.require_reliable and cal.reliability != "reliable" else 0
97
+
98
+
99
+ def cmd_ci(args: argparse.Namespace) -> int:
100
+ from .ci import render_markdown, run_ci, to_json, write_github_outputs
101
+
102
+ outcome = run_ci(
103
+ Database(args.db),
104
+ args.dataset,
105
+ args.baseline_agent,
106
+ args.candidate_agent,
107
+ concurrency=args.concurrency,
108
+ budget_usd=args.budget,
109
+ alpha=args.alpha,
110
+ )
111
+ markdown = render_markdown(outcome, args.baseline_agent, args.candidate_agent)
112
+ print(to_json(outcome))
113
+ write_github_outputs(outcome, markdown)
114
+ label = outcome.verdict.replace("_", " ").upper()
115
+ print(f"\n{label}", file=sys.stderr)
116
+ if outcome.verdict == "infrastructure_error":
117
+ return 2 # no verdict: never swallowed by --no-fail, because nothing was actually checked
118
+ return 0 if args.no_fail else outcome.exit_code_failure
119
+
120
+
121
+ def cmd_power(args: argparse.Namespace) -> int:
122
+ from .power import min_detectable_effect, power_for_effect, required_tasks
123
+
124
+ d, alpha, target = args.discordance, args.alpha, args.target_power
125
+ if args.effect is not None and args.tasks is not None:
126
+ p = power_for_effect(args.tasks, d, args.effect, alpha)
127
+ print(f"{args.tasks} tasks, {d:.0%} of tasks flipping: a {args.effect * 100:.1f}-point change is detected {p:.0%} of the time.")
128
+ elif args.effect is not None:
129
+ n = required_tasks(args.effect, d, alpha, target)
130
+ msg = f"about {n} tasks" if n else "more than 5000 tasks"
131
+ print(f"To detect a {args.effect * 100:.1f}-point change with {target:.0%} power (alpha {alpha}, {d:.0%} discordance) you need {msg}.")
132
+ elif args.tasks is not None:
133
+ mde = min_detectable_effect(args.tasks, d, alpha, target)
134
+ if mde is None:
135
+ print(f"{args.tasks} tasks cannot reach {target:.0%} power at {d:.0%} discordance, whatever the effect.")
136
+ else:
137
+ print(f"{args.tasks} tasks, {d:.0%} of tasks flipping: changes of {mde * 100:.1f} points or more are detected with {target:.0%} power.")
138
+ else:
139
+ print("give --tasks, --effect, or both", file=sys.stderr)
140
+ return 2
141
+ return 0
142
+
143
+
144
+ def cmd_stability(args: argparse.Namespace) -> int:
145
+ from .stability import analyze
146
+
147
+ db = Database(args.db)
148
+ dataset_id = _load_dataset(db, args.dataset)
149
+ ids = []
150
+ for i in range(args.repeats):
151
+ with db.session() as s:
152
+ run = Run(dataset_id=dataset_id, agent=args.agent, label=f"repeat {i + 1}", concurrency=args.concurrency)
153
+ s.add(run)
154
+ s.commit()
155
+ ids.append(run.id)
156
+ execute_run(db, ids[-1])
157
+ with db.session() as s:
158
+ runs = [list(s.get(Run, i).results) for i in ids]
159
+ report = analyze(runs)
160
+ print(json.dumps(report.to_dict(), indent=2))
161
+ print(
162
+ f"\n{report.flaky}/{report.tasks} tasks flaky ({report.flaky_fraction:.0%}); pass rate "
163
+ f"{report.mean_pass_rate:.1%} +/- {report.pass_rate_sd * 100:.1f} pts. Changes under "
164
+ f"~{report.noise_floor_pts:.1f} points are indistinguishable from re-running the same agent."
165
+ )
166
+ return 1 if args.max_flaky is not None and report.flaky_fraction > args.max_flaky else 0
167
+
168
+
169
+ def cmd_label(args: argparse.Namespace) -> int:
170
+ """Grade a sample of a run's answers by hand. The judge's verdict is never shown."""
171
+ from .calibration import label_sample
172
+ from .db import HumanLabel
173
+
174
+ db = Database(args.db)
175
+ with db.session() as s:
176
+ if s.get(Run, args.run) is None:
177
+ print("run not found", file=sys.stderr)
178
+ return 2
179
+ sample = label_sample(s, args.run, args.n)
180
+ if not sample:
181
+ print("nothing left to label (no LLM-judged, unlabeled answers in this run)")
182
+ return 0
183
+ done = 0
184
+ print(f"{len(sample)} answers to grade. [p]ass [f]ail [s]kip [q]uit and save\n")
185
+ for i, item in enumerate(sample, 1):
186
+ print(f"--- {i}/{len(sample)} " + "-" * 50)
187
+ print(f"QUESTION : {item['input']}")
188
+ if item["expected"]:
189
+ print(f"REFERENCE: {item['expected']}")
190
+ if item["rubric"]:
191
+ print(f"RUBRIC : {item['rubric']}")
192
+ print("ANSWER :\n" + "\n".join(" " + line for line in item["output"].splitlines()[: args.max_lines]))
193
+ while True:
194
+ try:
195
+ choice = input("grade> ").strip().lower()
196
+ except EOFError:
197
+ choice = "q"
198
+ if choice in ("p", "f", "s", "q"):
199
+ break
200
+ print("type p, f, s or q")
201
+ if choice == "q":
202
+ break
203
+ if choice == "s":
204
+ continue
205
+ row = s.query(HumanLabel).filter_by(run_id=args.run, task_id=item["task_id"]).first()
206
+ if row:
207
+ row.passed = choice == "p"
208
+ else:
209
+ s.add(HumanLabel(run_id=args.run, task_id=item["task_id"], passed=choice == "p"))
210
+ s.commit() # saved immediately: quitting or crashing never loses graded answers
211
+ done += 1
212
+ total = s.query(HumanLabel).filter_by(run_id=args.run).count()
213
+ print(f"\ngraded {done} this session, {total} in total. Next: agentprobe calibrate --run {args.run}")
214
+ return 0
215
+
216
+
217
+ def cmd_worker(args: argparse.Namespace) -> int:
218
+ import logging
219
+ import signal
220
+ import threading
221
+
222
+ from .jobqueue import RedisQueue
223
+ from .worker import Worker
224
+
225
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
226
+ stop = threading.Event()
227
+ for sig in (signal.SIGINT, signal.SIGTERM):
228
+ signal.signal(sig, lambda *_: stop.set()) # finish the current run, then exit
229
+ queue = RedisQueue.from_url(args.redis_url, lease_seconds=args.lease)
230
+ Worker(Database(args.db), queue).run_forever(stop)
231
+ return 0
232
+
233
+
234
+ def cmd_db(args: argparse.Namespace) -> int:
235
+ from . import migrate
236
+
237
+ db = Database(args.db, migrate=False)
238
+ if args.action == "upgrade":
239
+ migrate.upgrade(db.engine)
240
+ rev, head = migrate.current_revision(db.engine), migrate.head_revision()
241
+ print(f"revision {rev or 'none'} (latest {head})")
242
+ if args.action == "check":
243
+ diff = migrate.schema_diff(db.engine)
244
+ if rev != head or diff:
245
+ print("schema is not up to date: run `agentprobe db upgrade`" if rev != head else f"schema drift: {diff}")
246
+ return 1
247
+ return 0
248
+
249
+
250
+ def main(argv: list[str] | None = None) -> int:
251
+ parser = argparse.ArgumentParser(prog="agentprobe")
252
+ parser.add_argument("--db", default=None, help="database URL (default: sqlite:///agentprobe.db)")
253
+ sub = parser.add_subparsers(dest="command", required=True)
254
+
255
+ p_run = sub.add_parser("run", help="run an agent against a dataset file")
256
+ p_run.add_argument("--dataset", required=True)
257
+ p_run.add_argument("--agent", required=True)
258
+ p_run.add_argument("--label", default="")
259
+ p_run.add_argument("--concurrency", type=int, default=4)
260
+ p_run.add_argument("--budget", type=float, default=None, help="max spend in USD")
261
+ p_run.set_defaults(func=cmd_run)
262
+
263
+ p_cmp = sub.add_parser("compare", help="compare two runs")
264
+ p_cmp.add_argument("--baseline", type=int, required=True)
265
+ p_cmp.add_argument("--candidate", type=int, required=True)
266
+ p_cmp.add_argument("--alpha", type=float, default=0.05)
267
+ p_cmp.add_argument("--fail-on-regression", action="store_true")
268
+ p_cmp.set_defaults(func=cmd_compare)
269
+
270
+ p_cal = sub.add_parser("calibrate", help="measure an LLM judge's agreement with human labels")
271
+ p_cal.add_argument("--run", type=int, required=True)
272
+ p_cal.add_argument("--labels", help="JSON file of [{task_id, passed}] to import first")
273
+ p_cal.add_argument("--min-n", type=int, default=30)
274
+ p_cal.add_argument("--require-reliable", action="store_true", help="exit 1 unless the judge is reliable")
275
+ p_cal.set_defaults(func=cmd_calibrate)
276
+
277
+ p_ci = sub.add_parser("ci", help="run baseline + candidate agents and gate on the comparison")
278
+ p_ci.add_argument("--dataset", required=True)
279
+ p_ci.add_argument("--baseline-agent", required=True)
280
+ p_ci.add_argument("--candidate-agent", required=True)
281
+ p_ci.add_argument("--concurrency", type=int, default=4)
282
+ p_ci.add_argument("--budget", type=float, default=None, help="max spend in USD per run")
283
+ p_ci.add_argument("--alpha", type=float, default=0.05)
284
+ p_ci.add_argument("--no-fail", action="store_true", help="report a regression but exit 0")
285
+ p_ci.set_defaults(func=cmd_ci)
286
+
287
+ p_pw = sub.add_parser("power", help="how large a change can a dataset of N tasks detect?")
288
+ p_pw.add_argument("--tasks", type=int, help="dataset size")
289
+ p_pw.add_argument("--effect", type=float, help="true change in pass rate, e.g. 0.08 for 8 points")
290
+ p_pw.add_argument("--discordance", type=float, default=0.15, help="share of tasks whose outcome flips between versions")
291
+ p_pw.add_argument("--alpha", type=float, default=0.05)
292
+ p_pw.add_argument("--target-power", type=float, default=0.8)
293
+ p_pw.set_defaults(func=cmd_power)
294
+
295
+ p_st = sub.add_parser("stability", help="repeat an agent on a dataset and measure run-to-run noise")
296
+ p_st.add_argument("--dataset", required=True)
297
+ p_st.add_argument("--agent", required=True)
298
+ p_st.add_argument("--repeats", type=int, default=5)
299
+ p_st.add_argument("--concurrency", type=int, default=4)
300
+ p_st.add_argument("--max-flaky", type=float, default=None, help="exit 1 if more than this fraction of tasks is flaky")
301
+ p_st.set_defaults(func=cmd_stability)
302
+
303
+ p_lb = sub.add_parser("label", help="grade answers by hand to calibrate an LLM judge")
304
+ p_lb.add_argument("--run", type=int, required=True)
305
+ p_lb.add_argument("--n", type=int, default=40, help="how many answers to grade")
306
+ p_lb.add_argument("--max-lines", type=int, default=12)
307
+ p_lb.set_defaults(func=cmd_label)
308
+
309
+ p_wk = sub.add_parser("worker", help="run a queue worker that executes enqueued runs")
310
+ p_wk.add_argument("--redis-url", default=os.environ.get("REDIS_URL", "redis://localhost:6379/0"))
311
+ p_wk.add_argument("--lease", type=float, default=60.0, help="seconds before a silent worker's job is requeued")
312
+ p_wk.set_defaults(func=cmd_worker)
313
+
314
+ p_db = sub.add_parser("db", help="schema migrations: upgrade to the latest, show the revision, or check")
315
+ p_db.add_argument("action", choices=["upgrade", "current", "check"])
316
+ p_db.set_defaults(func=cmd_db)
317
+
318
+ args = parser.parse_args(argv)
319
+ from .telemetry import setup_telemetry
320
+
321
+ setup_telemetry("agentprobe-worker" if args.command == "worker" else "agentprobe")
322
+ return args.func(args)
323
+
324
+
325
+ if __name__ == "__main__":
326
+ sys.exit(main())
agentprobe/compare.py ADDED
@@ -0,0 +1,109 @@
1
+ """Statistical comparison of two runs on the same dataset.
2
+
3
+ Pass/fail outcomes are paired per task, so we use the exact McNemar test on the discordant
4
+ pairs rather than comparing two independent proportions. That is both more powerful and
5
+ correct for this design, and it needs nothing beyond the standard library.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import math
11
+ from dataclasses import asdict, dataclass
12
+
13
+ from .db import TaskResult
14
+
15
+
16
+ @dataclass
17
+ class RunStats:
18
+ n: int
19
+ pass_rate: float
20
+ mean_cost_usd: float
21
+ total_cost_usd: float
22
+ mean_latency_ms: float
23
+ p95_latency_ms: float
24
+ errors: int
25
+
26
+
27
+ @dataclass
28
+ class Comparison:
29
+ common_tasks: int
30
+ baseline: RunStats
31
+ candidate: RunStats
32
+ pass_rate_delta: float
33
+ fixed: int # failed in baseline, passes in candidate
34
+ broken: int # passed in baseline, fails in candidate
35
+ p_value: float
36
+ significant: bool
37
+ verdict: str # regression | improvement | no_significant_change
38
+
39
+ def to_dict(self) -> dict:
40
+ return asdict(self)
41
+
42
+
43
+ def mcnemar_exact(broken: int, fixed: int) -> float:
44
+ """Two-sided exact McNemar p-value from the discordant pair counts."""
45
+ n = broken + fixed
46
+ if n == 0:
47
+ return 1.0
48
+ k = min(broken, fixed)
49
+ tail = sum(math.comb(n, i) for i in range(k + 1)) / 2**n
50
+ return min(1.0, 2 * tail)
51
+
52
+
53
+ def percentile(values: list[float], q: float) -> float:
54
+ if not values:
55
+ return 0.0
56
+ ordered = sorted(values)
57
+ idx = max(0, math.ceil(q * len(ordered)) - 1)
58
+ return ordered[idx]
59
+
60
+
61
+ def run_stats(results: list[TaskResult]) -> RunStats:
62
+ n = len(results)
63
+ if n == 0:
64
+ return RunStats(0, 0.0, 0.0, 0.0, 0.0, 0.0, 0)
65
+ total_cost = sum(r.cost_usd for r in results)
66
+ latencies = [r.latency_ms for r in results if not r.error]
67
+ return RunStats(
68
+ n=n,
69
+ pass_rate=sum(r.passed for r in results) / n,
70
+ mean_cost_usd=total_cost / n,
71
+ total_cost_usd=total_cost,
72
+ mean_latency_ms=sum(latencies) / len(latencies) if latencies else 0.0,
73
+ p95_latency_ms=percentile(latencies, 0.95),
74
+ errors=sum(1 for r in results if r.error),
75
+ )
76
+
77
+
78
+ def compare_runs(
79
+ baseline: list[TaskResult], candidate: list[TaskResult], alpha: float = 0.05
80
+ ) -> Comparison:
81
+ base = {r.task_id: r for r in baseline}
82
+ cand = {r.task_id: r for r in candidate}
83
+ common = sorted(base.keys() & cand.keys())
84
+ b_res, c_res = [base[t] for t in common], [cand[t] for t in common]
85
+
86
+ broken = sum(1 for t in common if base[t].passed and not cand[t].passed)
87
+ fixed = sum(1 for t in common if not base[t].passed and cand[t].passed)
88
+ p = mcnemar_exact(broken, fixed)
89
+ significant = p < alpha
90
+
91
+ if significant and broken > fixed:
92
+ verdict = "regression"
93
+ elif significant and fixed > broken:
94
+ verdict = "improvement"
95
+ else:
96
+ verdict = "no_significant_change"
97
+
98
+ b_stats, c_stats = run_stats(b_res), run_stats(c_res)
99
+ return Comparison(
100
+ common_tasks=len(common),
101
+ baseline=b_stats,
102
+ candidate=c_stats,
103
+ pass_rate_delta=c_stats.pass_rate - b_stats.pass_rate,
104
+ fixed=fixed,
105
+ broken=broken,
106
+ p_value=p,
107
+ significant=significant,
108
+ verdict=verdict,
109
+ )