agentprobe-evals 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +3 -0
- agentprobe/agents.py +135 -0
- agentprobe/api.py +333 -0
- agentprobe/calibration.py +139 -0
- agentprobe/ci.py +216 -0
- agentprobe/cli.py +326 -0
- agentprobe/compare.py +109 -0
- agentprobe/dashboard/index.html +338 -0
- agentprobe/db.py +139 -0
- agentprobe/jobqueue.py +87 -0
- agentprobe/judges.py +95 -0
- agentprobe/llm_judge.py +169 -0
- agentprobe/migrate.py +77 -0
- agentprobe/migrations/env.py +15 -0
- agentprobe/migrations/script.py.mako +22 -0
- agentprobe/migrations/versions/0001_initial.py +83 -0
- agentprobe/power.py +122 -0
- agentprobe/runner.py +245 -0
- agentprobe/stability.py +64 -0
- agentprobe/telemetry.py +141 -0
- agentprobe/worker.py +81 -0
- agentprobe_evals-0.2.0.dist-info/METADATA +427 -0
- agentprobe_evals-0.2.0.dist-info/RECORD +27 -0
- agentprobe_evals-0.2.0.dist-info/WHEEL +5 -0
- agentprobe_evals-0.2.0.dist-info/entry_points.txt +2 -0
- agentprobe_evals-0.2.0.dist-info/licenses/LICENSE +21 -0
- agentprobe_evals-0.2.0.dist-info/top_level.txt +1 -0
agentprobe/ci.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""One-shot CI gate: run a baseline and a candidate agent, compare, report, and set an exit code.
|
|
2
|
+
|
|
3
|
+
Exit codes:
|
|
4
|
+
0 no regression (or --no-fail)
|
|
5
|
+
1 statistically significant regression
|
|
6
|
+
2 infrastructure problem (a run failed or too many tasks errored), so there is no verdict.
|
|
7
|
+
This is deliberately separate from 1: an outage in the agent's endpoint must not be
|
|
8
|
+
reported as "the change made the agent worse", and must not be silently treated as a pass.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import html
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
from sqlalchemy import select
|
|
19
|
+
|
|
20
|
+
from .compare import Comparison, compare_runs
|
|
21
|
+
from .db import Database, Run, Task, TaskResult
|
|
22
|
+
from .telemetry import tracer
|
|
23
|
+
|
|
24
|
+
MAX_ERROR_RATE = 0.2
|
|
25
|
+
OK_STATUSES = ("completed", "budget_exceeded")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class CIOutcome:
|
|
30
|
+
verdict: str # regression | improvement | no_significant_change | infrastructure_error
|
|
31
|
+
comparison: Comparison | None = None
|
|
32
|
+
problems: list[str] = field(default_factory=list)
|
|
33
|
+
broken: list[dict] = field(default_factory=list)
|
|
34
|
+
fixed: list[dict] = field(default_factory=list)
|
|
35
|
+
budget_limited: bool = False
|
|
36
|
+
alpha: float = 0.05
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def exit_code_failure(self) -> int:
|
|
40
|
+
return {"regression": 1, "infrastructure_error": 2}.get(self.verdict, 0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _flips(s, baseline: Run, candidate: Run) -> tuple[list[dict], list[dict]]:
|
|
44
|
+
tasks = {t.id: t for t in s.scalars(select(Task).where(Task.dataset_id == baseline.dataset_id))}
|
|
45
|
+
base = {r.task_id: r for r in baseline.results}
|
|
46
|
+
cand = {r.task_id: r for r in candidate.results}
|
|
47
|
+
|
|
48
|
+
def row(tid: int) -> dict:
|
|
49
|
+
return {
|
|
50
|
+
"input": tasks[tid].input,
|
|
51
|
+
"baseline": base[tid].output,
|
|
52
|
+
"candidate": cand[tid].output,
|
|
53
|
+
"reasoning": cand[tid].judge_reasoning,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
common = sorted(base.keys() & cand.keys())
|
|
57
|
+
broken = [row(t) for t in common if base[t].passed and not cand[t].passed]
|
|
58
|
+
fixed = [row(t) for t in common if not base[t].passed and cand[t].passed]
|
|
59
|
+
return broken, fixed
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _health(run: Run, results: list[TaskResult], name: str) -> list[str]:
|
|
63
|
+
problems = []
|
|
64
|
+
if run.status not in OK_STATUSES:
|
|
65
|
+
problems.append(f"{name} run ended as '{run.status}'" + (f": {run.error}" if run.error else ""))
|
|
66
|
+
if not results:
|
|
67
|
+
problems.append(f"{name} run produced no results")
|
|
68
|
+
else:
|
|
69
|
+
errors = sum(1 for r in results if r.error)
|
|
70
|
+
if errors / len(results) > MAX_ERROR_RATE:
|
|
71
|
+
sample = next(r.error for r in results if r.error)
|
|
72
|
+
problems.append(
|
|
73
|
+
f"{name}: {errors}/{len(results)} tasks errored (limit {MAX_ERROR_RATE:.0%}); e.g. {sample[:140]}"
|
|
74
|
+
)
|
|
75
|
+
return problems
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def run_ci(
|
|
79
|
+
db: Database,
|
|
80
|
+
dataset_path: str,
|
|
81
|
+
baseline_agent: str,
|
|
82
|
+
candidate_agent: str,
|
|
83
|
+
concurrency: int = 4,
|
|
84
|
+
budget_usd: float | None = None,
|
|
85
|
+
alpha: float = 0.05,
|
|
86
|
+
) -> CIOutcome:
|
|
87
|
+
from .cli import _load_dataset # same loader as `agentprobe run`
|
|
88
|
+
from .runner import execute_run
|
|
89
|
+
|
|
90
|
+
dataset_id = _load_dataset(db, dataset_path)
|
|
91
|
+
ids = {}
|
|
92
|
+
# One trace for the whole gate: ci -> (baseline run, candidate run) -> tasks -> agent/judge calls.
|
|
93
|
+
with tracer().start_as_current_span("agentprobe.ci", attributes={"agentprobe.alpha": alpha}):
|
|
94
|
+
for label, agent in (("baseline", baseline_agent), ("candidate", candidate_agent)):
|
|
95
|
+
with db.session() as s:
|
|
96
|
+
run = Run(
|
|
97
|
+
dataset_id=dataset_id, agent=agent, label=label, concurrency=concurrency, budget_usd=budget_usd
|
|
98
|
+
)
|
|
99
|
+
s.add(run)
|
|
100
|
+
s.commit()
|
|
101
|
+
ids[label] = run.id
|
|
102
|
+
execute_run(db, ids[label]) # sequential: the two runs must not compete for one endpoint
|
|
103
|
+
|
|
104
|
+
with db.session() as s:
|
|
105
|
+
b, c = s.get(Run, ids["baseline"]), s.get(Run, ids["candidate"])
|
|
106
|
+
problems = _health(b, list(b.results), "baseline") + _health(c, list(c.results), "candidate")
|
|
107
|
+
if problems:
|
|
108
|
+
return CIOutcome("infrastructure_error", problems=problems, alpha=alpha)
|
|
109
|
+
cmp = compare_runs(list(b.results), list(c.results), alpha)
|
|
110
|
+
broken, fixed = _flips(s, b, c)
|
|
111
|
+
limited = "budget_exceeded" in (b.status, c.status)
|
|
112
|
+
return CIOutcome(cmp.verdict, cmp, [], broken, fixed, limited, alpha)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# ---------- reporting ----------
|
|
116
|
+
|
|
117
|
+
def _cell(text: str, limit: int = 110) -> str:
|
|
118
|
+
"""Make untrusted text safe for a markdown table cell (agent output can contain anything)."""
|
|
119
|
+
flat = " ".join(str(text).split())
|
|
120
|
+
if len(flat) > limit:
|
|
121
|
+
flat = flat[: limit - 1] + "…"
|
|
122
|
+
return html.escape(flat).replace("|", "\\|").replace("`", "`")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _flip_table(rows: list[dict], title: str, max_rows: int = 10) -> str:
|
|
126
|
+
if not rows:
|
|
127
|
+
return ""
|
|
128
|
+
body = "\n".join(
|
|
129
|
+
f"| {_cell(r['input'], 80)} | {_cell(r['baseline'])} | {_cell(r['candidate'])} |" for r in rows[:max_rows]
|
|
130
|
+
)
|
|
131
|
+
more = f"\n\n…and {len(rows) - max_rows} more." if len(rows) > max_rows else ""
|
|
132
|
+
return (
|
|
133
|
+
f"<details><summary>{title} ({len(rows)})</summary>\n\n"
|
|
134
|
+
f"| Task | Baseline output | Candidate output |\n|---|---|---|\n{body}{more}\n\n</details>\n"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _sensitivity_line(c: Comparison) -> str:
|
|
139
|
+
"""Say how much this dataset can actually detect, so 'no significant change' is read correctly."""
|
|
140
|
+
from .power import min_detectable_effect
|
|
141
|
+
|
|
142
|
+
if c.common_tasks == 0:
|
|
143
|
+
return ""
|
|
144
|
+
observed = (c.broken + c.fixed) / c.common_tasks
|
|
145
|
+
discordance = min(1.0, max(observed, 0.05)) # floor: a quiet run must not claim infinite power
|
|
146
|
+
mde = min_detectable_effect(c.common_tasks, discordance)
|
|
147
|
+
basis = f"{c.common_tasks} tasks, about {discordance:.0%} of them flipping between versions"
|
|
148
|
+
if mde is None:
|
|
149
|
+
return f"<sub>Sensitivity: with {basis}, this comparison cannot reach 80% power, so a non-significant result says little.</sub>"
|
|
150
|
+
return f"<sub>Sensitivity: with {basis}, changes of {mde * 100:.0f} points or more are detected with 80% power.</sub>"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def render_markdown(o: CIOutcome, baseline_agent: str = "", candidate_agent: str = "") -> str:
|
|
154
|
+
if o.verdict == "infrastructure_error":
|
|
155
|
+
items = "\n".join(f"- {_cell(p, 300)}" for p in o.problems)
|
|
156
|
+
return (
|
|
157
|
+
"## AgentProbe: ⚠️ No verdict (infrastructure problem)\n\n"
|
|
158
|
+
"The comparison could not be made, so this is **not** a statement about the change.\n\n"
|
|
159
|
+
f"{items}\n"
|
|
160
|
+
)
|
|
161
|
+
c = o.comparison
|
|
162
|
+
assert c is not None
|
|
163
|
+
head = {
|
|
164
|
+
"regression": "❌ Regression detected",
|
|
165
|
+
"improvement": "✅ Improvement",
|
|
166
|
+
"no_significant_change": "✅ No significant change",
|
|
167
|
+
}[o.verdict]
|
|
168
|
+
delta = c.pass_rate_delta * 100
|
|
169
|
+
cost_b, cost_c = c.baseline.mean_cost_usd, c.candidate.mean_cost_usd
|
|
170
|
+
cost_row = (
|
|
171
|
+
f"| Mean cost / task | ${cost_b:.6f} | ${cost_c:.6f} | |\n" if (cost_b or cost_c) else ""
|
|
172
|
+
)
|
|
173
|
+
sig = "significant" if c.significant else "not significant"
|
|
174
|
+
out = (
|
|
175
|
+
f"## AgentProbe: {head}\n\n"
|
|
176
|
+
f"| | Baseline | Candidate | Δ |\n|---|---|---|---|\n"
|
|
177
|
+
f"| Pass rate | {c.baseline.pass_rate:.1%} | {c.candidate.pass_rate:.1%} | {delta:+.1f} pts |\n"
|
|
178
|
+
f"{cost_row}"
|
|
179
|
+
f"| p95 latency | {c.baseline.p95_latency_ms:.0f} ms | {c.candidate.p95_latency_ms:.0f} ms | |\n"
|
|
180
|
+
f"| Errored tasks | {c.baseline.errors} | {c.candidate.errors} | |\n\n"
|
|
181
|
+
f"**{c.broken} broke · {c.fixed} fixed** across {c.common_tasks} shared tasks. "
|
|
182
|
+
f"Exact McNemar p = {c.p_value:.4f} ({sig} at α = {o.alpha}).\n\n"
|
|
183
|
+
)
|
|
184
|
+
out += _sensitivity_line(c) + "\n\n"
|
|
185
|
+
if baseline_agent or candidate_agent:
|
|
186
|
+
out += f"<sub>baseline `{_cell(baseline_agent, 80)}` · candidate `{_cell(candidate_agent, 80)}`</sub>\n\n"
|
|
187
|
+
if o.budget_limited:
|
|
188
|
+
out += "> ⚠️ A run hit its cost budget and stopped early; only tasks both runs finished are compared.\n\n"
|
|
189
|
+
out += _flip_table(o.broken, "Tasks the candidate broke") + _flip_table(o.fixed, "Tasks the candidate fixed")
|
|
190
|
+
return out
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def write_github_outputs(o: CIOutcome, markdown: str) -> None:
|
|
194
|
+
"""Publish the job summary and step outputs when running under GitHub Actions."""
|
|
195
|
+
if path := os.environ.get("GITHUB_STEP_SUMMARY"):
|
|
196
|
+
with open(path, "a") as fh:
|
|
197
|
+
fh.write(markdown + "\n")
|
|
198
|
+
if path := os.environ.get("GITHUB_OUTPUT"):
|
|
199
|
+
c = o.comparison
|
|
200
|
+
values = {
|
|
201
|
+
"verdict": o.verdict,
|
|
202
|
+
"baseline-pass-rate": f"{c.baseline.pass_rate:.4f}" if c else "",
|
|
203
|
+
"candidate-pass-rate": f"{c.candidate.pass_rate:.4f}" if c else "",
|
|
204
|
+
"p-value": f"{c.p_value:.6f}" if c else "",
|
|
205
|
+
"broken": str(c.broken) if c else "",
|
|
206
|
+
"fixed": str(c.fixed) if c else "",
|
|
207
|
+
}
|
|
208
|
+
with open(path, "a") as fh:
|
|
209
|
+
fh.writelines(f"{key}={value}\n" for key, value in values.items())
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def to_json(o: CIOutcome) -> str:
|
|
213
|
+
return json.dumps(
|
|
214
|
+
{"verdict": o.verdict, "problems": o.problems, "comparison": o.comparison.to_dict() if o.comparison else None},
|
|
215
|
+
indent=2,
|
|
216
|
+
)
|
agentprobe/cli.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Command line interface, designed so CI can gate merges on agent quality.
|
|
2
|
+
|
|
3
|
+
agentprobe run --dataset examples/dataset.json --agent mock:accuracy=0.9 --label v1
|
|
4
|
+
agentprobe compare --baseline 1 --candidate 2 --fail-on-regression
|
|
5
|
+
agentprobe ci --dataset d.json --baseline-agent A --candidate-agent B # both steps, for CI
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from sqlalchemy import select
|
|
16
|
+
|
|
17
|
+
from .compare import compare_runs
|
|
18
|
+
from .db import Database, Dataset, Run, Task
|
|
19
|
+
from .runner import execute_run
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _load_dataset(db: Database, path: str) -> int:
|
|
23
|
+
with open(path) as fh:
|
|
24
|
+
data = json.load(fh)
|
|
25
|
+
with db.session() as s:
|
|
26
|
+
ds = s.scalar(select(Dataset).where(Dataset.name == data["name"]))
|
|
27
|
+
if ds is None:
|
|
28
|
+
ds = Dataset(name=data["name"], tasks=[Task(**t) for t in data["tasks"]])
|
|
29
|
+
s.add(ds)
|
|
30
|
+
s.commit()
|
|
31
|
+
return ds.id
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cmd_run(args: argparse.Namespace) -> int:
|
|
35
|
+
db = Database(args.db)
|
|
36
|
+
dataset_id = _load_dataset(db, args.dataset)
|
|
37
|
+
with db.session() as s:
|
|
38
|
+
run = Run(
|
|
39
|
+
dataset_id=dataset_id,
|
|
40
|
+
agent=args.agent,
|
|
41
|
+
label=args.label,
|
|
42
|
+
concurrency=args.concurrency,
|
|
43
|
+
budget_usd=args.budget,
|
|
44
|
+
)
|
|
45
|
+
s.add(run)
|
|
46
|
+
s.commit()
|
|
47
|
+
run_id = run.id
|
|
48
|
+
execute_run(db, run_id)
|
|
49
|
+
with db.session() as s:
|
|
50
|
+
run = s.get(Run, run_id)
|
|
51
|
+
passed = sum(r.passed for r in run.results)
|
|
52
|
+
print(f"run {run_id} [{run.status}] {passed}/{len(run.results)} passed ({args.agent})")
|
|
53
|
+
return 0 if run.status in ("completed", "budget_exceeded") else 1
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def cmd_compare(args: argparse.Namespace) -> int:
|
|
57
|
+
db = Database(args.db)
|
|
58
|
+
with db.session() as s:
|
|
59
|
+
b, c = s.get(Run, args.baseline), s.get(Run, args.candidate)
|
|
60
|
+
if b is None or c is None:
|
|
61
|
+
print("run not found", file=sys.stderr)
|
|
62
|
+
return 2
|
|
63
|
+
cmp = compare_runs(list(b.results), list(c.results), args.alpha)
|
|
64
|
+
print(json.dumps(cmp.to_dict(), indent=2))
|
|
65
|
+
print(
|
|
66
|
+
f"\n{cmp.verdict.upper()}: pass rate {cmp.baseline.pass_rate:.1%} -> "
|
|
67
|
+
f"{cmp.candidate.pass_rate:.1%} (broken={cmp.broken}, fixed={cmp.fixed}, p={cmp.p_value:.4f})"
|
|
68
|
+
)
|
|
69
|
+
return 1 if args.fail_on_regression and cmp.verdict == "regression" else 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def cmd_calibrate(args: argparse.Namespace) -> int:
|
|
73
|
+
from .calibration import run_calibration
|
|
74
|
+
from .db import HumanLabel
|
|
75
|
+
|
|
76
|
+
db = Database(args.db)
|
|
77
|
+
with db.session() as s:
|
|
78
|
+
if s.get(Run, args.run) is None:
|
|
79
|
+
print("run not found", file=sys.stderr)
|
|
80
|
+
return 2
|
|
81
|
+
if args.labels:
|
|
82
|
+
with open(args.labels) as fh:
|
|
83
|
+
labels = json.load(fh) # [{"task_id": 1, "passed": true}, ...]
|
|
84
|
+
existing = {
|
|
85
|
+
lb.task_id: lb for lb in s.scalars(select(HumanLabel).where(HumanLabel.run_id == args.run))
|
|
86
|
+
}
|
|
87
|
+
for item in labels:
|
|
88
|
+
if item["task_id"] in existing:
|
|
89
|
+
existing[item["task_id"]].passed = bool(item["passed"])
|
|
90
|
+
else:
|
|
91
|
+
s.add(HumanLabel(run_id=args.run, task_id=item["task_id"], passed=bool(item["passed"])))
|
|
92
|
+
s.commit()
|
|
93
|
+
cal = run_calibration(s, args.run, args.min_n)
|
|
94
|
+
print(json.dumps(cal.to_dict(), indent=2))
|
|
95
|
+
print(f"\n{cal.reliability.upper()}: kappa={cal.kappa:.2f}, agreement={cal.accuracy:.1%} on {cal.n} labels. {cal.note}")
|
|
96
|
+
return 1 if args.require_reliable and cal.reliability != "reliable" else 0
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def cmd_ci(args: argparse.Namespace) -> int:
|
|
100
|
+
from .ci import render_markdown, run_ci, to_json, write_github_outputs
|
|
101
|
+
|
|
102
|
+
outcome = run_ci(
|
|
103
|
+
Database(args.db),
|
|
104
|
+
args.dataset,
|
|
105
|
+
args.baseline_agent,
|
|
106
|
+
args.candidate_agent,
|
|
107
|
+
concurrency=args.concurrency,
|
|
108
|
+
budget_usd=args.budget,
|
|
109
|
+
alpha=args.alpha,
|
|
110
|
+
)
|
|
111
|
+
markdown = render_markdown(outcome, args.baseline_agent, args.candidate_agent)
|
|
112
|
+
print(to_json(outcome))
|
|
113
|
+
write_github_outputs(outcome, markdown)
|
|
114
|
+
label = outcome.verdict.replace("_", " ").upper()
|
|
115
|
+
print(f"\n{label}", file=sys.stderr)
|
|
116
|
+
if outcome.verdict == "infrastructure_error":
|
|
117
|
+
return 2 # no verdict: never swallowed by --no-fail, because nothing was actually checked
|
|
118
|
+
return 0 if args.no_fail else outcome.exit_code_failure
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def cmd_power(args: argparse.Namespace) -> int:
|
|
122
|
+
from .power import min_detectable_effect, power_for_effect, required_tasks
|
|
123
|
+
|
|
124
|
+
d, alpha, target = args.discordance, args.alpha, args.target_power
|
|
125
|
+
if args.effect is not None and args.tasks is not None:
|
|
126
|
+
p = power_for_effect(args.tasks, d, args.effect, alpha)
|
|
127
|
+
print(f"{args.tasks} tasks, {d:.0%} of tasks flipping: a {args.effect * 100:.1f}-point change is detected {p:.0%} of the time.")
|
|
128
|
+
elif args.effect is not None:
|
|
129
|
+
n = required_tasks(args.effect, d, alpha, target)
|
|
130
|
+
msg = f"about {n} tasks" if n else "more than 5000 tasks"
|
|
131
|
+
print(f"To detect a {args.effect * 100:.1f}-point change with {target:.0%} power (alpha {alpha}, {d:.0%} discordance) you need {msg}.")
|
|
132
|
+
elif args.tasks is not None:
|
|
133
|
+
mde = min_detectable_effect(args.tasks, d, alpha, target)
|
|
134
|
+
if mde is None:
|
|
135
|
+
print(f"{args.tasks} tasks cannot reach {target:.0%} power at {d:.0%} discordance, whatever the effect.")
|
|
136
|
+
else:
|
|
137
|
+
print(f"{args.tasks} tasks, {d:.0%} of tasks flipping: changes of {mde * 100:.1f} points or more are detected with {target:.0%} power.")
|
|
138
|
+
else:
|
|
139
|
+
print("give --tasks, --effect, or both", file=sys.stderr)
|
|
140
|
+
return 2
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def cmd_stability(args: argparse.Namespace) -> int:
|
|
145
|
+
from .stability import analyze
|
|
146
|
+
|
|
147
|
+
db = Database(args.db)
|
|
148
|
+
dataset_id = _load_dataset(db, args.dataset)
|
|
149
|
+
ids = []
|
|
150
|
+
for i in range(args.repeats):
|
|
151
|
+
with db.session() as s:
|
|
152
|
+
run = Run(dataset_id=dataset_id, agent=args.agent, label=f"repeat {i + 1}", concurrency=args.concurrency)
|
|
153
|
+
s.add(run)
|
|
154
|
+
s.commit()
|
|
155
|
+
ids.append(run.id)
|
|
156
|
+
execute_run(db, ids[-1])
|
|
157
|
+
with db.session() as s:
|
|
158
|
+
runs = [list(s.get(Run, i).results) for i in ids]
|
|
159
|
+
report = analyze(runs)
|
|
160
|
+
print(json.dumps(report.to_dict(), indent=2))
|
|
161
|
+
print(
|
|
162
|
+
f"\n{report.flaky}/{report.tasks} tasks flaky ({report.flaky_fraction:.0%}); pass rate "
|
|
163
|
+
f"{report.mean_pass_rate:.1%} +/- {report.pass_rate_sd * 100:.1f} pts. Changes under "
|
|
164
|
+
f"~{report.noise_floor_pts:.1f} points are indistinguishable from re-running the same agent."
|
|
165
|
+
)
|
|
166
|
+
return 1 if args.max_flaky is not None and report.flaky_fraction > args.max_flaky else 0
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def cmd_label(args: argparse.Namespace) -> int:
|
|
170
|
+
"""Grade a sample of a run's answers by hand. The judge's verdict is never shown."""
|
|
171
|
+
from .calibration import label_sample
|
|
172
|
+
from .db import HumanLabel
|
|
173
|
+
|
|
174
|
+
db = Database(args.db)
|
|
175
|
+
with db.session() as s:
|
|
176
|
+
if s.get(Run, args.run) is None:
|
|
177
|
+
print("run not found", file=sys.stderr)
|
|
178
|
+
return 2
|
|
179
|
+
sample = label_sample(s, args.run, args.n)
|
|
180
|
+
if not sample:
|
|
181
|
+
print("nothing left to label (no LLM-judged, unlabeled answers in this run)")
|
|
182
|
+
return 0
|
|
183
|
+
done = 0
|
|
184
|
+
print(f"{len(sample)} answers to grade. [p]ass [f]ail [s]kip [q]uit and save\n")
|
|
185
|
+
for i, item in enumerate(sample, 1):
|
|
186
|
+
print(f"--- {i}/{len(sample)} " + "-" * 50)
|
|
187
|
+
print(f"QUESTION : {item['input']}")
|
|
188
|
+
if item["expected"]:
|
|
189
|
+
print(f"REFERENCE: {item['expected']}")
|
|
190
|
+
if item["rubric"]:
|
|
191
|
+
print(f"RUBRIC : {item['rubric']}")
|
|
192
|
+
print("ANSWER :\n" + "\n".join(" " + line for line in item["output"].splitlines()[: args.max_lines]))
|
|
193
|
+
while True:
|
|
194
|
+
try:
|
|
195
|
+
choice = input("grade> ").strip().lower()
|
|
196
|
+
except EOFError:
|
|
197
|
+
choice = "q"
|
|
198
|
+
if choice in ("p", "f", "s", "q"):
|
|
199
|
+
break
|
|
200
|
+
print("type p, f, s or q")
|
|
201
|
+
if choice == "q":
|
|
202
|
+
break
|
|
203
|
+
if choice == "s":
|
|
204
|
+
continue
|
|
205
|
+
row = s.query(HumanLabel).filter_by(run_id=args.run, task_id=item["task_id"]).first()
|
|
206
|
+
if row:
|
|
207
|
+
row.passed = choice == "p"
|
|
208
|
+
else:
|
|
209
|
+
s.add(HumanLabel(run_id=args.run, task_id=item["task_id"], passed=choice == "p"))
|
|
210
|
+
s.commit() # saved immediately: quitting or crashing never loses graded answers
|
|
211
|
+
done += 1
|
|
212
|
+
total = s.query(HumanLabel).filter_by(run_id=args.run).count()
|
|
213
|
+
print(f"\ngraded {done} this session, {total} in total. Next: agentprobe calibrate --run {args.run}")
|
|
214
|
+
return 0
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def cmd_worker(args: argparse.Namespace) -> int:
|
|
218
|
+
import logging
|
|
219
|
+
import signal
|
|
220
|
+
import threading
|
|
221
|
+
|
|
222
|
+
from .jobqueue import RedisQueue
|
|
223
|
+
from .worker import Worker
|
|
224
|
+
|
|
225
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
226
|
+
stop = threading.Event()
|
|
227
|
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
228
|
+
signal.signal(sig, lambda *_: stop.set()) # finish the current run, then exit
|
|
229
|
+
queue = RedisQueue.from_url(args.redis_url, lease_seconds=args.lease)
|
|
230
|
+
Worker(Database(args.db), queue).run_forever(stop)
|
|
231
|
+
return 0
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def cmd_db(args: argparse.Namespace) -> int:
|
|
235
|
+
from . import migrate
|
|
236
|
+
|
|
237
|
+
db = Database(args.db, migrate=False)
|
|
238
|
+
if args.action == "upgrade":
|
|
239
|
+
migrate.upgrade(db.engine)
|
|
240
|
+
rev, head = migrate.current_revision(db.engine), migrate.head_revision()
|
|
241
|
+
print(f"revision {rev or 'none'} (latest {head})")
|
|
242
|
+
if args.action == "check":
|
|
243
|
+
diff = migrate.schema_diff(db.engine)
|
|
244
|
+
if rev != head or diff:
|
|
245
|
+
print("schema is not up to date: run `agentprobe db upgrade`" if rev != head else f"schema drift: {diff}")
|
|
246
|
+
return 1
|
|
247
|
+
return 0
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def main(argv: list[str] | None = None) -> int:
|
|
251
|
+
parser = argparse.ArgumentParser(prog="agentprobe")
|
|
252
|
+
parser.add_argument("--db", default=None, help="database URL (default: sqlite:///agentprobe.db)")
|
|
253
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
254
|
+
|
|
255
|
+
p_run = sub.add_parser("run", help="run an agent against a dataset file")
|
|
256
|
+
p_run.add_argument("--dataset", required=True)
|
|
257
|
+
p_run.add_argument("--agent", required=True)
|
|
258
|
+
p_run.add_argument("--label", default="")
|
|
259
|
+
p_run.add_argument("--concurrency", type=int, default=4)
|
|
260
|
+
p_run.add_argument("--budget", type=float, default=None, help="max spend in USD")
|
|
261
|
+
p_run.set_defaults(func=cmd_run)
|
|
262
|
+
|
|
263
|
+
p_cmp = sub.add_parser("compare", help="compare two runs")
|
|
264
|
+
p_cmp.add_argument("--baseline", type=int, required=True)
|
|
265
|
+
p_cmp.add_argument("--candidate", type=int, required=True)
|
|
266
|
+
p_cmp.add_argument("--alpha", type=float, default=0.05)
|
|
267
|
+
p_cmp.add_argument("--fail-on-regression", action="store_true")
|
|
268
|
+
p_cmp.set_defaults(func=cmd_compare)
|
|
269
|
+
|
|
270
|
+
p_cal = sub.add_parser("calibrate", help="measure an LLM judge's agreement with human labels")
|
|
271
|
+
p_cal.add_argument("--run", type=int, required=True)
|
|
272
|
+
p_cal.add_argument("--labels", help="JSON file of [{task_id, passed}] to import first")
|
|
273
|
+
p_cal.add_argument("--min-n", type=int, default=30)
|
|
274
|
+
p_cal.add_argument("--require-reliable", action="store_true", help="exit 1 unless the judge is reliable")
|
|
275
|
+
p_cal.set_defaults(func=cmd_calibrate)
|
|
276
|
+
|
|
277
|
+
p_ci = sub.add_parser("ci", help="run baseline + candidate agents and gate on the comparison")
|
|
278
|
+
p_ci.add_argument("--dataset", required=True)
|
|
279
|
+
p_ci.add_argument("--baseline-agent", required=True)
|
|
280
|
+
p_ci.add_argument("--candidate-agent", required=True)
|
|
281
|
+
p_ci.add_argument("--concurrency", type=int, default=4)
|
|
282
|
+
p_ci.add_argument("--budget", type=float, default=None, help="max spend in USD per run")
|
|
283
|
+
p_ci.add_argument("--alpha", type=float, default=0.05)
|
|
284
|
+
p_ci.add_argument("--no-fail", action="store_true", help="report a regression but exit 0")
|
|
285
|
+
p_ci.set_defaults(func=cmd_ci)
|
|
286
|
+
|
|
287
|
+
p_pw = sub.add_parser("power", help="how large a change can a dataset of N tasks detect?")
|
|
288
|
+
p_pw.add_argument("--tasks", type=int, help="dataset size")
|
|
289
|
+
p_pw.add_argument("--effect", type=float, help="true change in pass rate, e.g. 0.08 for 8 points")
|
|
290
|
+
p_pw.add_argument("--discordance", type=float, default=0.15, help="share of tasks whose outcome flips between versions")
|
|
291
|
+
p_pw.add_argument("--alpha", type=float, default=0.05)
|
|
292
|
+
p_pw.add_argument("--target-power", type=float, default=0.8)
|
|
293
|
+
p_pw.set_defaults(func=cmd_power)
|
|
294
|
+
|
|
295
|
+
p_st = sub.add_parser("stability", help="repeat an agent on a dataset and measure run-to-run noise")
|
|
296
|
+
p_st.add_argument("--dataset", required=True)
|
|
297
|
+
p_st.add_argument("--agent", required=True)
|
|
298
|
+
p_st.add_argument("--repeats", type=int, default=5)
|
|
299
|
+
p_st.add_argument("--concurrency", type=int, default=4)
|
|
300
|
+
p_st.add_argument("--max-flaky", type=float, default=None, help="exit 1 if more than this fraction of tasks is flaky")
|
|
301
|
+
p_st.set_defaults(func=cmd_stability)
|
|
302
|
+
|
|
303
|
+
p_lb = sub.add_parser("label", help="grade answers by hand to calibrate an LLM judge")
|
|
304
|
+
p_lb.add_argument("--run", type=int, required=True)
|
|
305
|
+
p_lb.add_argument("--n", type=int, default=40, help="how many answers to grade")
|
|
306
|
+
p_lb.add_argument("--max-lines", type=int, default=12)
|
|
307
|
+
p_lb.set_defaults(func=cmd_label)
|
|
308
|
+
|
|
309
|
+
p_wk = sub.add_parser("worker", help="run a queue worker that executes enqueued runs")
|
|
310
|
+
p_wk.add_argument("--redis-url", default=os.environ.get("REDIS_URL", "redis://localhost:6379/0"))
|
|
311
|
+
p_wk.add_argument("--lease", type=float, default=60.0, help="seconds before a silent worker's job is requeued")
|
|
312
|
+
p_wk.set_defaults(func=cmd_worker)
|
|
313
|
+
|
|
314
|
+
p_db = sub.add_parser("db", help="schema migrations: upgrade to the latest, show the revision, or check")
|
|
315
|
+
p_db.add_argument("action", choices=["upgrade", "current", "check"])
|
|
316
|
+
p_db.set_defaults(func=cmd_db)
|
|
317
|
+
|
|
318
|
+
args = parser.parse_args(argv)
|
|
319
|
+
from .telemetry import setup_telemetry
|
|
320
|
+
|
|
321
|
+
setup_telemetry("agentprobe-worker" if args.command == "worker" else "agentprobe")
|
|
322
|
+
return args.func(args)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
if __name__ == "__main__":
|
|
326
|
+
sys.exit(main())
|
agentprobe/compare.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Statistical comparison of two runs on the same dataset.
|
|
2
|
+
|
|
3
|
+
Pass/fail outcomes are paired per task, so we use the exact McNemar test on the discordant
|
|
4
|
+
pairs rather than comparing two independent proportions. That is both more powerful and
|
|
5
|
+
correct for this design, and it needs nothing beyond the standard library.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from dataclasses import asdict, dataclass
|
|
12
|
+
|
|
13
|
+
from .db import TaskResult
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class RunStats:
|
|
18
|
+
n: int
|
|
19
|
+
pass_rate: float
|
|
20
|
+
mean_cost_usd: float
|
|
21
|
+
total_cost_usd: float
|
|
22
|
+
mean_latency_ms: float
|
|
23
|
+
p95_latency_ms: float
|
|
24
|
+
errors: int
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Comparison:
|
|
29
|
+
common_tasks: int
|
|
30
|
+
baseline: RunStats
|
|
31
|
+
candidate: RunStats
|
|
32
|
+
pass_rate_delta: float
|
|
33
|
+
fixed: int # failed in baseline, passes in candidate
|
|
34
|
+
broken: int # passed in baseline, fails in candidate
|
|
35
|
+
p_value: float
|
|
36
|
+
significant: bool
|
|
37
|
+
verdict: str # regression | improvement | no_significant_change
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict:
|
|
40
|
+
return asdict(self)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def mcnemar_exact(broken: int, fixed: int) -> float:
|
|
44
|
+
"""Two-sided exact McNemar p-value from the discordant pair counts."""
|
|
45
|
+
n = broken + fixed
|
|
46
|
+
if n == 0:
|
|
47
|
+
return 1.0
|
|
48
|
+
k = min(broken, fixed)
|
|
49
|
+
tail = sum(math.comb(n, i) for i in range(k + 1)) / 2**n
|
|
50
|
+
return min(1.0, 2 * tail)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def percentile(values: list[float], q: float) -> float:
|
|
54
|
+
if not values:
|
|
55
|
+
return 0.0
|
|
56
|
+
ordered = sorted(values)
|
|
57
|
+
idx = max(0, math.ceil(q * len(ordered)) - 1)
|
|
58
|
+
return ordered[idx]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run_stats(results: list[TaskResult]) -> RunStats:
|
|
62
|
+
n = len(results)
|
|
63
|
+
if n == 0:
|
|
64
|
+
return RunStats(0, 0.0, 0.0, 0.0, 0.0, 0.0, 0)
|
|
65
|
+
total_cost = sum(r.cost_usd for r in results)
|
|
66
|
+
latencies = [r.latency_ms for r in results if not r.error]
|
|
67
|
+
return RunStats(
|
|
68
|
+
n=n,
|
|
69
|
+
pass_rate=sum(r.passed for r in results) / n,
|
|
70
|
+
mean_cost_usd=total_cost / n,
|
|
71
|
+
total_cost_usd=total_cost,
|
|
72
|
+
mean_latency_ms=sum(latencies) / len(latencies) if latencies else 0.0,
|
|
73
|
+
p95_latency_ms=percentile(latencies, 0.95),
|
|
74
|
+
errors=sum(1 for r in results if r.error),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def compare_runs(
|
|
79
|
+
baseline: list[TaskResult], candidate: list[TaskResult], alpha: float = 0.05
|
|
80
|
+
) -> Comparison:
|
|
81
|
+
base = {r.task_id: r for r in baseline}
|
|
82
|
+
cand = {r.task_id: r for r in candidate}
|
|
83
|
+
common = sorted(base.keys() & cand.keys())
|
|
84
|
+
b_res, c_res = [base[t] for t in common], [cand[t] for t in common]
|
|
85
|
+
|
|
86
|
+
broken = sum(1 for t in common if base[t].passed and not cand[t].passed)
|
|
87
|
+
fixed = sum(1 for t in common if not base[t].passed and cand[t].passed)
|
|
88
|
+
p = mcnemar_exact(broken, fixed)
|
|
89
|
+
significant = p < alpha
|
|
90
|
+
|
|
91
|
+
if significant and broken > fixed:
|
|
92
|
+
verdict = "regression"
|
|
93
|
+
elif significant and fixed > broken:
|
|
94
|
+
verdict = "improvement"
|
|
95
|
+
else:
|
|
96
|
+
verdict = "no_significant_change"
|
|
97
|
+
|
|
98
|
+
b_stats, c_stats = run_stats(b_res), run_stats(c_res)
|
|
99
|
+
return Comparison(
|
|
100
|
+
common_tasks=len(common),
|
|
101
|
+
baseline=b_stats,
|
|
102
|
+
candidate=c_stats,
|
|
103
|
+
pass_rate_delta=c_stats.pass_rate - b_stats.pass_rate,
|
|
104
|
+
fixed=fixed,
|
|
105
|
+
broken=broken,
|
|
106
|
+
p_value=p,
|
|
107
|
+
significant=significant,
|
|
108
|
+
verdict=verdict,
|
|
109
|
+
)
|