agentprobe-evals 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentprobe/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """AgentProbe: evaluation and regression testing for LLM agents."""
2
+
3
+ __version__ = "0.2.0"
agentprobe/agents.py ADDED
@@ -0,0 +1,135 @@
1
+ """Agent adapters.
2
+
3
+ An agent is anything with `run(task_input) -> AgentResult`. Agents are selected by a
4
+ spec string so a run is fully described by plain text and can be reproduced later:
5
+
6
+ mock:accuracy=0.9,seed=1,latency=25 deterministic simulated agent (demos, tests, CI)
7
+ mock:accuracy=0.9,jitter=0.3 same, but 30% of calls are re-drawn: a flaky agent
8
+ http:https://host/path POSTs {"input": ...}, expects AgentResult-shaped JSON
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import hashlib
14
+ import random
15
+ import threading
16
+ import time
17
+ from dataclasses import dataclass, field
18
+ from typing import Protocol
19
+
20
+ import httpx
21
+ from opentelemetry import propagate
22
+
23
+
24
+ @dataclass
25
+ class AgentResult:
26
+ output: str
27
+ tool_calls: list[str] = field(default_factory=list)
28
+ input_tokens: int = 0
29
+ output_tokens: int = 0
30
+ cost_usd: float = 0.0
31
+
32
+
33
+ class Agent(Protocol):
34
+ def run(self, task_input: str, expected: str = "") -> AgentResult: ...
35
+
36
+
37
+ class MockAgent:
38
+ """Simulated agent that answers correctly with a fixed probability.
39
+
40
+ Correctness is a deterministic function of (seed, input), so reruns are identical and
41
+ two mock agents with different accuracy form a realistic baseline/candidate pair.
42
+ It needs `expected` to know what a correct answer looks like.
43
+ """
44
+
45
+ PRICE_IN = 3e-6
46
+ PRICE_OUT = 15e-6
47
+
48
+ def __init__(
49
+ self,
50
+ accuracy: float = 0.8,
51
+ seed: int = 0,
52
+ tools: str = "search",
53
+ latency_ms: float = 0.0,
54
+ jitter: float = 0.0,
55
+ rng_seed: int | None = None,
56
+ ) -> None:
57
+ if not 0.0 <= accuracy <= 1.0:
58
+ raise ValueError("accuracy must be between 0 and 1")
59
+ self.accuracy = accuracy
60
+ self.seed = seed
61
+ self.tools = [t for t in tools.split("|") if t]
62
+ self.latency_ms = latency_ms
63
+ if not 0.0 <= jitter <= 1.0:
64
+ raise ValueError("jitter must be between 0 and 1")
65
+ # NOTE: rng_seed makes ONE agent instance reproducible; every run builds a fresh instance,
66
+ # so repeated runs with the same rng_seed replay the same noise. Omit it to get real variation.
67
+ # jitter = chance that a call ignores its input-determined outcome and re-draws at random,
68
+ # which makes repeated runs differ the way a real LLM agent does.
69
+ self.jitter = jitter
70
+ self._rng = random.Random(rng_seed)
71
+ self._rng_lock = threading.Lock()
72
+
73
+ def _roll(self, task_input: str) -> float:
74
+ digest = hashlib.sha256(f"{self.seed}:{task_input}".encode()).digest()
75
+ return int.from_bytes(digest[:8], "big") / 2**64
76
+
77
+ def run(self, task_input: str, expected: str = "") -> AgentResult:
78
+ if self.latency_ms: # simulated, deterministic jitter between 0.5x and 1.5x
79
+ time.sleep(self.latency_ms * (0.5 + self._roll("latency:" + task_input)) / 1000)
80
+ correct = self._roll(task_input) < self.accuracy
81
+ if self.jitter:
82
+ with self._rng_lock:
83
+ if self._rng.random() < self.jitter:
84
+ correct = self._rng.random() < self.accuracy
85
+ output = expected if correct else "???"
86
+ in_tok = max(1, len(task_input) // 4)
87
+ out_tok = max(1, len(output) // 4)
88
+ return AgentResult(
89
+ output=output,
90
+ tool_calls=list(self.tools),
91
+ input_tokens=in_tok,
92
+ output_tokens=out_tok,
93
+ cost_usd=in_tok * self.PRICE_IN + out_tok * self.PRICE_OUT,
94
+ )
95
+
96
+
97
+ class HTTPAgent:
98
+ """Calls a remote agent endpoint. Response JSON may contain any AgentResult field."""
99
+
100
+ def __init__(self, url: str, timeout: float = 60.0) -> None:
101
+ self.url = url
102
+ self.timeout = timeout
103
+
104
+ def run(self, task_input: str, expected: str = "") -> AgentResult:
105
+ headers: dict[str, str] = {}
106
+ propagate.inject(headers) # lets an instrumented agent join this trace via `traceparent`
107
+ resp = httpx.post(self.url, json={"input": task_input}, headers=headers, timeout=self.timeout)
108
+ resp.raise_for_status()
109
+ data = resp.json()
110
+ return AgentResult(
111
+ output=str(data.get("output", "")),
112
+ tool_calls=list(data.get("tool_calls", [])),
113
+ input_tokens=int(data.get("input_tokens", 0)),
114
+ output_tokens=int(data.get("output_tokens", 0)),
115
+ cost_usd=float(data.get("cost_usd", 0.0)),
116
+ )
117
+
118
+
119
+ def build_agent(spec: str) -> Agent:
120
+ kind, _, rest = spec.partition(":")
121
+ if kind == "mock":
122
+ params = dict(p.split("=", 1) for p in rest.split(",") if p)
123
+ return MockAgent(
124
+ accuracy=float(params.get("accuracy", 0.8)),
125
+ seed=int(params.get("seed", 0)),
126
+ tools=params.get("tools", "search"),
127
+ latency_ms=float(params.get("latency", 0)),
128
+ jitter=float(params.get("jitter", 0)),
129
+ rng_seed=int(params["rng"]) if "rng" in params else None,
130
+ )
131
+ if kind == "http":
132
+ if not rest:
133
+ raise ValueError("http agent needs a URL: http:https://host/path")
134
+ return HTTPAgent(rest)
135
+ raise ValueError(f"unknown agent spec: {spec!r}")
agentprobe/api.py ADDED
@@ -0,0 +1,333 @@
1
+ """REST API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hmac
6
+ import logging
7
+ import os
8
+ from pathlib import Path
9
+
10
+ from fastapi import APIRouter, BackgroundTasks, Depends, FastAPI, HTTPException, Request
11
+ from fastapi.responses import HTMLResponse
12
+ from pydantic import BaseModel, Field
13
+ from sqlalchemy import select
14
+
15
+ from . import __version__
16
+ from .agents import build_agent
17
+ from .calibration import label_sample, run_calibration
18
+ from .compare import compare_runs, run_stats
19
+ from .db import Database, Dataset, HumanLabel, Run, Task, TaskResult
20
+ from .jobqueue import RedisQueue
21
+ from .judges import JUDGE_NAMES
22
+ from .runner import execute_run, fail_run
23
+ from .telemetry import agent_kind, capture_context, setup_telemetry, tracer
24
+
25
+ DASHBOARD_HTML = Path(__file__).parent / "dashboard" / "index.html"
26
+ log = logging.getLogger("agentprobe.api")
27
+
28
+
29
+ def _split_keys(value: str | None) -> list[str]:
30
+ return [k.strip() for k in (value or "").split(",") if k.strip()]
31
+
32
+
33
+ def _bearer(request: Request) -> str:
34
+ header = request.headers.get("authorization", "")
35
+ scheme, _, token = header.partition(" ")
36
+ if scheme.lower() == "bearer" and token:
37
+ return token.strip()
38
+ return request.headers.get("x-api-key", "").strip()
39
+
40
+
41
+ def _matches(candidate: str, keys: list[str]) -> bool:
42
+ # compare against every key (no early exit) in constant time
43
+ ok = False
44
+ for key in keys:
45
+ ok |= hmac.compare_digest(candidate.encode(), key.encode())
46
+ return ok
47
+
48
+
49
+ class TaskIn(BaseModel):
50
+ input: str
51
+ expected: str = ""
52
+ judge: str = "contains"
53
+ judge_config: dict = Field(default_factory=dict)
54
+
55
+
56
+ class DatasetIn(BaseModel):
57
+ name: str = Field(min_length=1, max_length=200)
58
+ tasks: list[TaskIn] = Field(min_length=1)
59
+
60
+
61
+ class LabelIn(BaseModel):
62
+ task_id: int
63
+ passed: bool
64
+
65
+
66
+ class LabelsIn(BaseModel):
67
+ labels: list[LabelIn] = Field(min_length=1)
68
+
69
+
70
+ class RunIn(BaseModel):
71
+ dataset_id: int
72
+ agent: str
73
+ label: str = ""
74
+ concurrency: int = Field(default=4, ge=1, le=64)
75
+ retries: int = Field(default=1, ge=0, le=5)
76
+ budget_usd: float | None = Field(default=None, gt=0)
77
+
78
+
79
+ def create_app(
80
+ database_url: str | None = None,
81
+ run_inline: bool = False,
82
+ queue: RedisQueue | None = None,
83
+ api_keys: list[str] | None = None,
84
+ read_keys: list[str] | None = None,
85
+ ) -> FastAPI:
86
+ """App factory.
87
+
88
+ Execution mode: `run_inline` runs synchronously (tests); a `queue` hands runs to separate
89
+ worker processes; with neither, runs execute in a background task of this process.
90
+
91
+ Authentication: `api_keys` (default: env AGENTPROBE_API_KEYS, comma-separated) grant full access;
92
+ `read_keys` (env AGENTPROBE_READ_API_KEYS) may only issue GET requests. Send a key as
93
+ `Authorization: Bearer <key>` or `X-API-Key`. With no keys configured the API is open (fine on
94
+ localhost; the server logs a warning) unless AGENTPROBE_REQUIRE_AUTH=1, which refuses to start.
95
+ `/health` and the dashboard page (which holds no data) are always public.
96
+ """
97
+ setup_telemetry() # no-op unless OTEL_EXPORTER_OTLP_ENDPOINT / AGENTPROBE_OTEL_CONSOLE is set
98
+ db = Database(database_url)
99
+ app = FastAPI(title="AgentProbe", version=__version__)
100
+
101
+ full_keys = api_keys if api_keys is not None else _split_keys(os.environ.get("AGENTPROBE_API_KEYS"))
102
+ ro_keys = read_keys if read_keys is not None else _split_keys(os.environ.get("AGENTPROBE_READ_API_KEYS"))
103
+ auth_enabled = bool(full_keys or ro_keys)
104
+ if not auth_enabled:
105
+ if os.environ.get("AGENTPROBE_REQUIRE_AUTH", "").lower() in ("1", "true", "yes"):
106
+ raise RuntimeError("AGENTPROBE_REQUIRE_AUTH is set but no AGENTPROBE_API_KEYS are configured")
107
+ log.warning("API authentication is disabled: set AGENTPROBE_API_KEYS to require an API key")
108
+
109
+ def require_auth(request: Request) -> None:
110
+ if not auth_enabled:
111
+ return
112
+ key = _bearer(request)
113
+ if key and _matches(key, full_keys):
114
+ return
115
+ if key and _matches(key, ro_keys):
116
+ if request.method in ("GET", "HEAD", "OPTIONS"):
117
+ return
118
+ raise HTTPException(403, "this API key is read-only")
119
+ raise HTTPException(401, "missing or invalid API key", headers={"WWW-Authenticate": "Bearer"})
120
+
121
+ api = APIRouter(dependencies=[Depends(require_auth)])
122
+
123
+ @app.get("/health")
124
+ def health() -> dict:
125
+ info: dict = {"status": "ok", "version": __version__, "execution": "inline", "auth": auth_enabled}
126
+ if queue is not None:
127
+ info["execution"] = "queue"
128
+ try:
129
+ info["queue"] = queue.stats()
130
+ except Exception: # noqa: BLE001
131
+ info["status"] = "degraded"
132
+ info["queue"] = "unreachable"
133
+ elif not run_inline:
134
+ info["execution"] = "in-process"
135
+ return info
136
+
137
+ @api.post("/datasets", status_code=201)
138
+ def create_dataset(body: DatasetIn) -> dict:
139
+ for t in body.tasks:
140
+ if t.judge not in JUDGE_NAMES:
141
+ raise HTTPException(422, f"unknown judge {t.judge!r}; available: {JUDGE_NAMES}")
142
+ with db.session() as s:
143
+ if s.scalar(select(Dataset).where(Dataset.name == body.name)):
144
+ raise HTTPException(409, f"dataset {body.name!r} already exists")
145
+ ds = Dataset(name=body.name, tasks=[Task(**t.model_dump()) for t in body.tasks])
146
+ s.add(ds)
147
+ s.commit()
148
+ return {"id": ds.id, "name": ds.name, "tasks": len(ds.tasks)}
149
+
150
+ @api.get("/datasets")
151
+ def list_datasets() -> list[dict]:
152
+ with db.session() as s:
153
+ return [
154
+ {"id": d.id, "name": d.name, "tasks": len(d.tasks)}
155
+ for d in s.scalars(select(Dataset).order_by(Dataset.id))
156
+ ]
157
+
158
+ @api.post("/runs", status_code=202)
159
+ def create_run(body: RunIn, background: BackgroundTasks) -> dict:
160
+ with tracer().start_as_current_span(
161
+ "agentprobe.create_run",
162
+ attributes={"agentprobe.dataset.id": body.dataset_id, "agentprobe.agent.kind": agent_kind(body.agent)},
163
+ ):
164
+ # The trace context is stored on the run so a worker in another process, possibly
165
+ # minutes later, continues the same trace.
166
+ return _create_run(body, background, capture_context())
167
+
168
+ def _create_run(body: RunIn, background: BackgroundTasks, trace_ctx: str) -> dict:
169
+ try:
170
+ build_agent(body.agent) # validate the spec up front
171
+ except ValueError as exc:
172
+ raise HTTPException(422, str(exc)) from exc
173
+ with db.session() as s:
174
+ if s.get(Dataset, body.dataset_id) is None:
175
+ raise HTTPException(404, "dataset not found")
176
+ run = Run(**body.model_dump(), trace_context=trace_ctx)
177
+ s.add(run)
178
+ s.commit()
179
+ run_id = run.id
180
+ if run_inline:
181
+ execute_run(db, run_id)
182
+ elif queue is not None:
183
+ try:
184
+ queue.enqueue(run_id)
185
+ except Exception as exc:
186
+ fail_run(db, run_id, f"could not enqueue: {exc}")
187
+ raise HTTPException(503, "job queue unavailable") from exc
188
+ else:
189
+ background.add_task(execute_run, db, run_id)
190
+ return {"id": run_id, "status": "pending"}
191
+
192
+ @api.get("/runs")
193
+ def list_runs(dataset_id: int | None = None, limit: int = 100) -> list[dict]:
194
+ """Most recent `limit` runs, oldest first so clients can plot them as a time series."""
195
+ with db.session() as s:
196
+ q = select(Run).order_by(Run.id.desc()).limit(max(1, min(limit, 500)))
197
+ if dataset_id is not None:
198
+ q = q.where(Run.dataset_id == dataset_id)
199
+ runs = list(s.scalars(q))[::-1]
200
+ n_tasks = {
201
+ d.id: len(d.tasks) for d in s.scalars(select(Dataset)) if d.id in {r.dataset_id for r in runs}
202
+ }
203
+ return [_run_summary(r, n_tasks.get(r.dataset_id, 0)) for r in runs]
204
+
205
+ @api.get("/runs/{run_id}")
206
+ def get_run(run_id: int) -> dict:
207
+ with db.session() as s:
208
+ run = _get_run(s, run_id)
209
+ return _run_summary(run, len(s.get(Dataset, run.dataset_id).tasks))
210
+
211
+ @api.get("/runs/{run_id}/results")
212
+ def get_results(run_id: int) -> list[dict]:
213
+ with db.session() as s:
214
+ run = _get_run(s, run_id)
215
+ tasks = {t.id: t for t in s.scalars(select(Task).where(Task.dataset_id == run.dataset_id))}
216
+ return [
217
+ {
218
+ "task_id": r.task_id,
219
+ "input": tasks[r.task_id].input,
220
+ "expected": tasks[r.task_id].expected,
221
+ "judge": tasks[r.task_id].judge,
222
+ "passed": r.passed,
223
+ "output": r.output,
224
+ "latency_ms": round(r.latency_ms, 2),
225
+ "cost_usd": r.cost_usd,
226
+ "attempts": r.attempts,
227
+ "error": r.error,
228
+ "judge_reasoning": r.judge_reasoning,
229
+ }
230
+ for r in sorted(run.results, key=lambda r: r.task_id)
231
+ ]
232
+
233
+ @api.get("/compare")
234
+ def compare(
235
+ baseline: int, candidate: int, alpha: float = 0.05, details: bool = False
236
+ ) -> dict:
237
+ with db.session() as s:
238
+ b, c = _get_run(s, baseline), _get_run(s, candidate)
239
+ if b.dataset_id != c.dataset_id:
240
+ raise HTTPException(422, "runs were made on different datasets")
241
+ out = compare_runs(list(b.results), list(c.results), alpha).to_dict()
242
+ if details:
243
+ out.update(_flipped_tasks(s, b, c))
244
+ return out
245
+
246
+ @api.get("/runs/{run_id}/label-sample")
247
+ def get_label_sample(run_id: int, n: int = 30) -> list[dict]:
248
+ with db.session() as s:
249
+ _get_run(s, run_id)
250
+ return label_sample(s, run_id, max(1, min(n, 200)))
251
+
252
+ @api.post("/runs/{run_id}/labels")
253
+ def post_labels(run_id: int, body: LabelsIn) -> dict:
254
+ with db.session() as s:
255
+ _get_run(s, run_id)
256
+ have = set(s.scalars(select(TaskResult.task_id).where(TaskResult.run_id == run_id)))
257
+ unknown = sorted({lb.task_id for lb in body.labels} - have)
258
+ if unknown:
259
+ raise HTTPException(422, f"tasks without a result in this run: {unknown}")
260
+ existing = {
261
+ lb.task_id: lb for lb in s.scalars(select(HumanLabel).where(HumanLabel.run_id == run_id))
262
+ }
263
+ for lb in body.labels: # upsert: re-labeling corrects an earlier mistake
264
+ if lb.task_id in existing:
265
+ existing[lb.task_id].passed = lb.passed
266
+ else:
267
+ s.add(HumanLabel(run_id=run_id, task_id=lb.task_id, passed=lb.passed))
268
+ s.commit()
269
+ return {"labeled": s.query(HumanLabel).filter_by(run_id=run_id).count()}
270
+
271
+ @api.get("/runs/{run_id}/calibration")
272
+ def get_calibration(run_id: int, min_n: int = 30) -> dict:
273
+ with db.session() as s:
274
+ _get_run(s, run_id)
275
+ return run_calibration(s, run_id, min_n).to_dict()
276
+
277
+ def _run_summary(run: Run, n_tasks: int) -> dict:
278
+ return {
279
+ "id": run.id,
280
+ "dataset_id": run.dataset_id,
281
+ "label": run.label,
282
+ "agent": run.agent,
283
+ "status": run.status,
284
+ "error": run.error,
285
+ "created_at": run.created_at.isoformat(),
286
+ "tasks_total": n_tasks,
287
+ "budget_usd": run.budget_usd,
288
+ "stats": run_stats(list(run.results)).__dict__,
289
+ }
290
+
291
+ def _flipped_tasks(s, b: Run, c: Run) -> dict:
292
+ tasks = {t.id: t for t in s.scalars(select(Task).where(Task.dataset_id == b.dataset_id))}
293
+ base, cand = {r.task_id: r for r in b.results}, {r.task_id: r for r in c.results}
294
+
295
+ def row(tid: int) -> dict:
296
+ return {
297
+ "task_id": tid,
298
+ "input": tasks[tid].input,
299
+ "expected": tasks[tid].expected,
300
+ "baseline_output": base[tid].output,
301
+ "candidate_output": cand[tid].output,
302
+ "candidate_reasoning": cand[tid].judge_reasoning,
303
+ }
304
+
305
+ common = sorted(base.keys() & cand.keys())
306
+ return {
307
+ "broken_tasks": [row(t) for t in common if base[t].passed and not cand[t].passed],
308
+ "fixed_tasks": [row(t) for t in common if not base[t].passed and cand[t].passed],
309
+ }
310
+
311
+ @app.get("/", include_in_schema=False)
312
+ def dashboard() -> HTMLResponse:
313
+ return HTMLResponse(DASHBOARD_HTML.read_text(encoding="utf-8"))
314
+
315
+ def _get_run(s, run_id: int) -> Run:
316
+ run = s.get(Run, run_id)
317
+ if run is None:
318
+ raise HTTPException(404, f"run {run_id} not found")
319
+ return run
320
+
321
+ app.include_router(api)
322
+ app.state.db = db
323
+ app.state.auth_enabled = auth_enabled
324
+ return app
325
+
326
+
327
+ def app_factory() -> FastAPI:
328
+ """Entry point for `uvicorn agentprobe.api:app_factory --factory`."""
329
+ redis_url = os.environ.get("REDIS_URL")
330
+ return create_app(queue=RedisQueue.from_url(redis_url) if redis_url else None)
331
+
332
+
333
+ __all__ = ["TaskResult", "app_factory", "create_app"]
@@ -0,0 +1,139 @@
1
+ """Calibrating a judge against human labels.
2
+
3
+ An LLM judge is a measuring instrument; before trusting it to gate a merge you measure how often
4
+ it agrees with people. We report raw accuracy (with a Wilson interval, because label sets are
5
+ small), precision/recall/specificity, and Cohen's kappa, which corrects agreement for chance.
6
+ Kappa matters because with 90% of answers passing, a judge that always says "pass" is 90%
7
+ "accurate" and completely useless.
8
+
9
+ Convention: "pass" is the positive class. Human labels are the ground truth.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ import random
16
+ from dataclasses import asdict, dataclass
17
+
18
+ from sqlalchemy import select
19
+ from sqlalchemy.orm import Session
20
+
21
+ from .db import HumanLabel, Task, TaskResult
22
+
23
+
24
+ @dataclass
25
+ class Calibration:
26
+ n: int
27
+ tp: int # judge pass, human pass
28
+ fp: int # judge pass, human fail (judge too lenient)
29
+ tn: int
30
+ fn: int # judge fail, human pass (judge too harsh)
31
+ accuracy: float
32
+ accuracy_ci: tuple[float, float]
33
+ precision: float | None
34
+ recall: float | None
35
+ specificity: float | None
36
+ kappa: float
37
+ reliability: str # reliable | use_with_caution | unreliable | insufficient_data
38
+ note: str
39
+
40
+ def to_dict(self) -> dict:
41
+ return asdict(self)
42
+
43
+
44
+ def wilson_interval(successes: int, n: int, z: float = 1.96) -> tuple[float, float]:
45
+ if n == 0:
46
+ return (0.0, 1.0)
47
+ p = successes / n
48
+ denom = 1 + z * z / n
49
+ centre = (p + z * z / (2 * n)) / denom
50
+ half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
51
+ return (max(0.0, centre - half), min(1.0, centre + half))
52
+
53
+
54
+ def cohens_kappa(tp: int, fp: int, tn: int, fn: int) -> float:
55
+ n = tp + fp + tn + fn
56
+ if n == 0:
57
+ return 0.0
58
+ observed = (tp + tn) / n
59
+ expected = ((tp + fp) * (tp + fn) + (fn + tn) * (fp + tn)) / (n * n)
60
+ if expected == 1.0: # both raters constant and identical: agreement is perfect but untestable
61
+ return 1.0 if observed == 1.0 else 0.0
62
+ return (observed - expected) / (1 - expected)
63
+
64
+
65
+ def _ratio(a: int, b: int) -> float | None:
66
+ return a / b if b else None
67
+
68
+
69
+ def calibrate(pairs: list[tuple[bool, bool]], min_n: int = 30) -> Calibration:
70
+ """`pairs` is a list of (human_passed, judge_passed)."""
71
+ tp = sum(1 for h, j in pairs if h and j)
72
+ fp = sum(1 for h, j in pairs if not h and j)
73
+ tn = sum(1 for h, j in pairs if not h and not j)
74
+ fn = sum(1 for h, j in pairs if h and not j)
75
+ n = len(pairs)
76
+ kappa = cohens_kappa(tp, fp, tn, fn)
77
+ accuracy = (tp + tn) / n if n else 0.0
78
+
79
+ humans_pass = tp + fn
80
+ if n < min_n:
81
+ reliability, note = "insufficient_data", f"only {n} labels; collect at least {min_n}"
82
+ elif humans_pass in (0, n):
83
+ reliability = "insufficient_data"
84
+ note = "all human labels are the same class, so kappa cannot separate judge from chance"
85
+ elif kappa >= 0.7:
86
+ reliability, note = "reliable", "substantial agreement with humans"
87
+ elif kappa >= 0.4:
88
+ reliability, note = "use_with_caution", "moderate agreement; review disagreements, refine the rubric"
89
+ else:
90
+ reliability, note = "unreliable", "weak agreement; do not gate on this judge"
91
+ if reliability != "insufficient_data" and fp > fn * 2 and fp >= 3:
92
+ note += "; leans lenient (many false passes)"
93
+ elif reliability != "insufficient_data" and fn > fp * 2 and fn >= 3:
94
+ note += "; leans harsh (many false fails)"
95
+
96
+ return Calibration(
97
+ n=n, tp=tp, fp=fp, tn=tn, fn=fn,
98
+ accuracy=accuracy,
99
+ accuracy_ci=wilson_interval(tp + tn, n),
100
+ precision=_ratio(tp, tp + fp),
101
+ recall=_ratio(tp, tp + fn),
102
+ specificity=_ratio(tn, tn + fp),
103
+ kappa=kappa,
104
+ reliability=reliability,
105
+ note=note,
106
+ )
107
+
108
+
109
+ def run_calibration(s: Session, run_id: int, min_n: int = 30) -> Calibration:
110
+ """Judge-vs-human agreement for one run, over its labeled, LLM-judged, non-errored tasks."""
111
+ rows = s.execute(
112
+ select(HumanLabel.passed, TaskResult.passed)
113
+ .join(TaskResult, (TaskResult.run_id == HumanLabel.run_id) & (TaskResult.task_id == HumanLabel.task_id))
114
+ .join(Task, Task.id == HumanLabel.task_id)
115
+ .where(HumanLabel.run_id == run_id, Task.judge == "llm", TaskResult.error == "")
116
+ ).all()
117
+ return calibrate([(h, j) for h, j in rows], min_n)
118
+
119
+
120
+ def label_sample(s: Session, run_id: int, n: int) -> list[dict]:
121
+ """Unlabeled LLM-judged answers for a person to grade.
122
+
123
+ The judge's verdict is deliberately withheld so it cannot anchor the human. The shuffle is
124
+ seeded by the run id, so repeated calls return a stable sample.
125
+ """
126
+ labeled = set(s.scalars(select(HumanLabel.task_id).where(HumanLabel.run_id == run_id)))
127
+ rows = s.execute(
128
+ select(Task.id, Task.input, Task.expected, TaskResult.output, Task.judge_config)
129
+ .join(TaskResult, TaskResult.task_id == Task.id)
130
+ .where(TaskResult.run_id == run_id, Task.judge == "llm", TaskResult.error == "")
131
+ .order_by(Task.id)
132
+ ).all()
133
+ pool = [
134
+ {"task_id": tid, "input": inp, "expected": exp, "output": out, "rubric": (cfg or {}).get("rubric", "")}
135
+ for tid, inp, exp, out, cfg in rows
136
+ if tid not in labeled
137
+ ]
138
+ random.Random(run_id).shuffle(pool)
139
+ return pool[:n]