agentprobe-evals 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe_evals-0.2.0/LICENSE +21 -0
- agentprobe_evals-0.2.0/PKG-INFO +427 -0
- agentprobe_evals-0.2.0/README.md +385 -0
- agentprobe_evals-0.2.0/pyproject.toml +59 -0
- agentprobe_evals-0.2.0/setup.cfg +4 -0
- agentprobe_evals-0.2.0/src/agentprobe/__init__.py +3 -0
- agentprobe_evals-0.2.0/src/agentprobe/agents.py +135 -0
- agentprobe_evals-0.2.0/src/agentprobe/api.py +333 -0
- agentprobe_evals-0.2.0/src/agentprobe/calibration.py +139 -0
- agentprobe_evals-0.2.0/src/agentprobe/ci.py +216 -0
- agentprobe_evals-0.2.0/src/agentprobe/cli.py +326 -0
- agentprobe_evals-0.2.0/src/agentprobe/compare.py +109 -0
- agentprobe_evals-0.2.0/src/agentprobe/dashboard/index.html +338 -0
- agentprobe_evals-0.2.0/src/agentprobe/db.py +139 -0
- agentprobe_evals-0.2.0/src/agentprobe/jobqueue.py +87 -0
- agentprobe_evals-0.2.0/src/agentprobe/judges.py +95 -0
- agentprobe_evals-0.2.0/src/agentprobe/llm_judge.py +169 -0
- agentprobe_evals-0.2.0/src/agentprobe/migrate.py +77 -0
- agentprobe_evals-0.2.0/src/agentprobe/migrations/env.py +15 -0
- agentprobe_evals-0.2.0/src/agentprobe/migrations/script.py.mako +22 -0
- agentprobe_evals-0.2.0/src/agentprobe/migrations/versions/0001_initial.py +83 -0
- agentprobe_evals-0.2.0/src/agentprobe/power.py +122 -0
- agentprobe_evals-0.2.0/src/agentprobe/runner.py +245 -0
- agentprobe_evals-0.2.0/src/agentprobe/stability.py +64 -0
- agentprobe_evals-0.2.0/src/agentprobe/telemetry.py +141 -0
- agentprobe_evals-0.2.0/src/agentprobe/worker.py +81 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/PKG-INFO +427 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/SOURCES.txt +43 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/dependency_links.txt +1 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/entry_points.txt +2 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/requires.txt +23 -0
- agentprobe_evals-0.2.0/src/agentprobe_evals.egg-info/top_level.txt +1 -0
- agentprobe_evals-0.2.0/tests/test_auth.py +106 -0
- agentprobe_evals-0.2.0/tests/test_ci.py +101 -0
- agentprobe_evals-0.2.0/tests/test_core.py +157 -0
- agentprobe_evals-0.2.0/tests/test_dashboard.py +63 -0
- agentprobe_evals-0.2.0/tests/test_docs_rag_datasets.py +48 -0
- agentprobe_evals-0.2.0/tests/test_label.py +85 -0
- agentprobe_evals-0.2.0/tests/test_llm_judge.py +268 -0
- agentprobe_evals-0.2.0/tests/test_migrations.py +71 -0
- agentprobe_evals-0.2.0/tests/test_otlp_wire.py +119 -0
- agentprobe_evals-0.2.0/tests/test_power_stability.py +138 -0
- agentprobe_evals-0.2.0/tests/test_queue.py +236 -0
- agentprobe_evals-0.2.0/tests/test_redis_integration.py +53 -0
- agentprobe_evals-0.2.0/tests/test_telemetry.py +207 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Amit Ginzberg
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agentprobe-evals
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Evaluation and regression-testing platform for LLM agents
|
|
5
|
+
Author: Amit Ginzberg
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/amitginz/agentprobe
|
|
8
|
+
Project-URL: Source, https://github.com/amitginz/agentprobe
|
|
9
|
+
Project-URL: Issues, https://github.com/amitginz/agentprobe/issues
|
|
10
|
+
Keywords: llm,agents,evaluation,regression-testing,mcnemar,llm-as-judge,opentelemetry
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: fastapi>=0.110
|
|
22
|
+
Requires-Dist: uvicorn>=0.29
|
|
23
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
24
|
+
Requires-Dist: pydantic>=2.6
|
|
25
|
+
Requires-Dist: httpx>=0.27
|
|
26
|
+
Requires-Dist: redis>=5.0
|
|
27
|
+
Requires-Dist: opentelemetry-api>=1.25
|
|
28
|
+
Requires-Dist: alembic>=1.13
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
32
|
+
Requires-Dist: fakeredis>=2.23; extra == "dev"
|
|
33
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
34
|
+
Requires-Dist: opentelemetry-sdk>=1.25; extra == "dev"
|
|
35
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.25; extra == "dev"
|
|
36
|
+
Provides-Extra: otel
|
|
37
|
+
Requires-Dist: opentelemetry-sdk>=1.25; extra == "otel"
|
|
38
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.25; extra == "otel"
|
|
39
|
+
Provides-Extra: postgres
|
|
40
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == "postgres"
|
|
41
|
+
Dynamic: license-file
|
|
42
|
+
|
|
43
|
+
# AgentProbe
|
|
44
|
+
|
|
45
|
+
[](https://github.com/amitginz/agentprobe/actions/workflows/ci.yml)
|
|
46
|
+
|
|
47
|
+
Regression testing for LLM agents. Run an agent against a dataset, record a full trace of every
|
|
48
|
+
task (output, tool calls, tokens, latency, cost), and answer the question that matters before you
|
|
49
|
+
ship a prompt or model change: **did it get better, worse, or is that just noise?**
|
|
50
|
+
|
|
51
|
+
```text
|
|
52
|
+
$ agentprobe run --dataset examples/dataset.json --agent "mock:accuracy=0.95" --label v1
|
|
53
|
+
run 1 [completed] 38/40 passed
|
|
54
|
+
$ agentprobe run --dataset examples/dataset.json --agent "mock:accuracy=0.60" --label v2
|
|
55
|
+
run 2 [completed] 29/40 passed
|
|
56
|
+
$ agentprobe compare --baseline 1 --candidate 2 --fail-on-regression
|
|
57
|
+
REGRESSION: pass rate 95.0% -> 72.5% (broken=9, fixed=0, p=0.0039)
|
|
58
|
+
$ echo $?
|
|
59
|
+
1
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The non-zero exit code is the point: drop the last command into CI and a PR that degrades your
|
|
63
|
+
agent fails the build, the same way a failing unit test would.
|
|
64
|
+
|
|
65
|
+
## Why not just compare pass rates?
|
|
66
|
+
|
|
67
|
+
Agent evals are small and noisy. "92% vs 88%" on 50 tasks is usually nothing. Because both runs
|
|
68
|
+
answer the *same* tasks, AgentProbe pairs the outcomes and applies an **exact McNemar test** to the
|
|
69
|
+
tasks that flipped (broke vs. fixed). A single flaky task does not trigger a regression; twelve
|
|
70
|
+
broken tasks and zero fixed ones does. The test is implemented in ~10 lines of stdlib code
|
|
71
|
+
([`compare.py`](src/agentprobe/compare.py)) and unit-tested against known values.
|
|
72
|
+
|
|
73
|
+
## Features
|
|
74
|
+
|
|
75
|
+
| Area | What is implemented |
|
|
76
|
+
|---|---|
|
|
77
|
+
| Execution | Thread-pool runner with bounded concurrency, per-task retries, and a **hard cost budget** that stops a run early (`budget_exceeded`) |
|
|
78
|
+
| Judges | Deterministic: `exact`, `contains`, `regex`, `tool_called`. **`llm`**: rubric-based LLM-as-judge with injection-resistant prompting, strict JSON verdicts, cost tracking |
|
|
79
|
+
| Judge calibration | Label a sample by hand and measure the judge: agreement with a Wilson CI, precision/recall, **Cohen's kappa**, a reliability verdict, and a lenient/harsh bias note |
|
|
80
|
+
| Agents | Spec-string adapters: `mock:` (deterministic simulated agent) and `http:` (any remote agent that returns JSON) |
|
|
81
|
+
| Analysis | Paired comparison: pass-rate delta, broken/fixed counts, exact p-value, cost and p95 latency |
|
|
82
|
+
| API | FastAPI: datasets, runs (async, background execution), results, `/compare` |
|
|
83
|
+
| Dashboard | Zero-build web UI at `/`: KPI tiles, pass-rate / cost / p95 trend charts (inline SVG), run history, run drill-down, and a side-by-side **regression view** listing exactly which tasks a change broke or fixed; auto-refreshes while runs are in flight |
|
|
84
|
+
| Queue | **Redis job queue with leases and heartbeats**: at-least-once delivery, crash recovery that resumes unfinished runs, poison-job protection, horizontally scalable `agentprobe worker` processes |
|
|
85
|
+
| Storage | SQLAlchemy 2.0; SQLite by default, PostgreSQL via `DATABASE_URL`; **Alembic migrations** applied automatically, existing databases adopted safely (`agentprobe db check` as a deploy gate) |
|
|
86
|
+
| Security | Optional **API keys** (full and read-only scopes, rotation, constant-time check); `AGENTPROBE_REQUIRE_AUTH=1` refuses to start without keys |
|
|
87
|
+
| Observability | **OpenTelemetry** traces and metrics: one trace per run (`create_run` → `run` → `task` → `agent.call` / `judge.evaluate`), context carried API → queue → worker and into HTTP agents, OTLP export, Jaeger in docker-compose |
|
|
88
|
+
| Ops | Dockerfile, docker-compose (API + workers + Redis + Postgres), **reusable GitHub Action** that gates PRs on agent quality |
|
|
89
|
+
|
|
90
|
+
Try the dashboard with seeded history (a bad model swap is hidden in there for you to find):
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
python examples/seed_demo.py && uvicorn agentprobe.api:app_factory --factory # http://localhost:8000
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Install from PyPI (the package is `agentprobe-evals`; the command and import are `agentprobe`):
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
pip install agentprobe-evals # extras: [otel], [postgres]
|
|
100
|
+
agentprobe run --dataset dataset.json --agent "mock:accuracy=0.95" --label v1
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Run the service from source:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pip install -e ".[dev]"
|
|
107
|
+
uvicorn agentprobe.api:app_factory --factory --reload # http://localhost:8000/docs
|
|
108
|
+
docker compose up --build # API + 2 workers + Redis + PostgreSQL
|
|
109
|
+
pytest
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Secure it (otherwise the API is open, which is fine on localhost):
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
export AGENTPROBE_API_KEYS=$(openssl rand -hex 24) # full access
|
|
116
|
+
export AGENTPROBE_READ_API_KEYS=$(openssl rand -hex 24) # GET only, e.g. for the dashboard
|
|
117
|
+
export AGENTPROBE_REQUIRE_AUTH=1 # refuse to start without a key
|
|
118
|
+
curl -H "Authorization: Bearer $AGENTPROBE_API_KEYS" localhost:8000/runs
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
The dashboard asks for a key when the server requires one. Keys are shared secrets, not user
|
|
122
|
+
accounts; put the API behind TLS. Schema changes ship as migrations: `agentprobe db upgrade`.
|
|
123
|
+
|
|
124
|
+
Execution modes: with `REDIS_URL` set the API only enqueues runs and returns `202`; any number of
|
|
125
|
+
`agentprobe worker` processes (on any machine that can reach Redis and the database) execute them.
|
|
126
|
+
Without it, runs execute in a background task of the API process, which is handy for local use.
|
|
127
|
+
If a worker is killed mid-run, its lease expires, another worker picks the run up, and it
|
|
128
|
+
**resumes from the tasks already finished** rather than starting over or re-spending the budget.
|
|
129
|
+
|
|
130
|
+
Evaluate your own agent by exposing an endpoint that accepts `{"input": "..."}` and returns
|
|
131
|
+
`{"output": "...", "tool_calls": [...], "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}`:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
agentprobe run --dataset my_tasks.json --agent "http:http://localhost:9000/agent" --budget 2.50
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Use it as a GitHub Action
|
|
138
|
+
|
|
139
|
+
Gate pull requests on agent quality. The action runs a baseline and a candidate agent on the same
|
|
140
|
+
dataset, compares them with the paired exact test, writes a report to the job summary, and fails
|
|
141
|
+
the job on a statistically significant regression:
|
|
142
|
+
|
|
143
|
+
```yaml
|
|
144
|
+
# .github/workflows/agent-quality.yml
|
|
145
|
+
on: pull_request
|
|
146
|
+
jobs:
|
|
147
|
+
agent-quality:
|
|
148
|
+
runs-on: ubuntu-latest
|
|
149
|
+
steps:
|
|
150
|
+
- uses: actions/checkout@v4
|
|
151
|
+
- uses: amitginz/agentprobe@v0.1.0
|
|
152
|
+
with:
|
|
153
|
+
dataset: evals/support-bot.json
|
|
154
|
+
baseline-agent: "http:https://staging.example.com/agent" # what is deployed
|
|
155
|
+
candidate-agent: "http:https://pr-${{ github.event.number }}.preview.example.com/agent"
|
|
156
|
+
budget-usd: "2.00"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The job summary shows pass rate, cost and p95 for both sides, how many tasks broke or were fixed,
|
|
160
|
+
the p-value, and a table of the tasks that broke with both outputs. Outputs (`verdict`,
|
|
161
|
+
`baseline-pass-rate`, `candidate-pass-rate`, `p-value`, `broken`, `fixed`) can drive later steps.
|
|
162
|
+
Inputs: `fail-on-regression`, `alpha`, `concurrency`, `budget-usd`, `python-version`.
|
|
163
|
+
|
|
164
|
+
Design choices worth knowing:
|
|
165
|
+
|
|
166
|
+
- **Three outcomes, not two.** Exit 0 means no regression, exit 1 means a significant regression,
|
|
167
|
+
exit 2 means **no verdict** because an endpoint was down or over 20% of tasks errored.
|
|
168
|
+
An outage is never reported as "your change made the agent worse", and never passes silently
|
|
169
|
+
either; `fail-on-regression: false` does not hide it.
|
|
170
|
+
- **Inputs never touch the shell.** They reach the script as environment variables, so a crafted
|
|
171
|
+
input cannot inject commands (a test asserts the step contains no `${{ }}` interpolation).
|
|
172
|
+
- **Untrusted output is escaped.** Agent output is HTML-escaped and flattened before it goes into
|
|
173
|
+
the summary table, so it cannot break the table or inject markup.
|
|
174
|
+
- **Same logic locally:** `agentprobe ci --dataset d.json --baseline-agent A --candidate-agent B`.
|
|
175
|
+
|
|
176
|
+
This repo uses its own action in [`ci.yml`](.github/workflows/ci.yml), and that job runs green on
|
|
177
|
+
GitHub-hosted runners. The CI test job also runs the Redis integration test against a real Redis
|
|
178
|
+
service (62 tests, none skipped).
|
|
179
|
+
|
|
180
|
+
## Observability (OpenTelemetry)
|
|
181
|
+
|
|
182
|
+
Every run is traceable. Point AgentProbe at any OTLP/HTTP collector and each evaluation becomes
|
|
183
|
+
a trace you can open in Jaeger, Tempo, Honeycomb or Datadog:
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
pip install "agentprobe-evals[otel]"
|
|
187
|
+
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 agentprobe ci --dataset ... --baseline-agent ... --candidate-agent ...
|
|
188
|
+
docker compose up # API + workers + Jaeger UI on http://localhost:16686
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
```text
|
|
192
|
+
agentprobe.create_run API request
|
|
193
|
+
└── agentprobe.run one per execution (a resumed run gets a new span in the same trace)
|
|
194
|
+
└── agentprobe.task one per task, correctly parented across the thread pool
|
|
195
|
+
├── agent.call one per attempt; failures carry the exception and ERROR status
|
|
196
|
+
└── judge.evaluate
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
- **Context survives process boundaries.** The W3C trace context is stored on the run row, so a
|
|
200
|
+
worker that picks the job up from Redis (even minutes later, after a crash and re-delivery)
|
|
201
|
+
continues the API request's trace. For `http:` agents it also sends a `traceparent` header, so an
|
|
202
|
+
instrumented agent under test shows up as child spans of the evaluation.
|
|
203
|
+
- **Metrics:** `agentprobe.tasks` (by judge / passed / errored), `agentprobe.agent.latency`
|
|
204
|
+
histogram and `agentprobe.cost`.
|
|
205
|
+
- **Privacy by default.** Task inputs, agent outputs and judge reasoning are never put on spans,
|
|
206
|
+
only sizes, token counts, costs, verdicts and timings, and an agent spec is reduced to its kind
|
|
207
|
+
(`http`), since spec URLs can embed credentials. A test asserts neither leaks.
|
|
208
|
+
- **Zero cost when off.** Only the tiny `opentelemetry-api` is a core dependency (a no-op until an
|
|
209
|
+
SDK is configured); exporting needs the `otel` extra.
|
|
210
|
+
- Verified end to end over the real OTLP wire format (a test runs the CLI in a separate process
|
|
211
|
+
against a local collector and decodes the protobuf), **and against a real Jaeger 2.23** (official
|
|
212
|
+
release binary): one `agentprobe ci` comparison of 20 tasks x 2 runs arrives as a single
|
|
213
|
+
126-span trace of depth 4, and Jaeger's GenAI view recognises the 40 `agent.call` spans from their
|
|
214
|
+
`gen_ai.usage.*` attributes. The docker-compose file itself has still not been run.
|
|
215
|
+
- **Errors are visible but not leaky.** A failed attempt is a red `agent.call` span with the
|
|
216
|
+
exception type; URLs in the message are redacted and no stack trace is exported. (This came from
|
|
217
|
+
looking at a real trace: the first Jaeger run showed the agent endpoint URL inside an HTTP error
|
|
218
|
+
message, which an earlier test of mine had missed because it only inspected span attributes.)
|
|
219
|
+
- Traces-only backends such as Jaeger reject OTLP metrics; set `OTEL_METRICS_EXPORTER=none`
|
|
220
|
+
(the compose file does).
|
|
221
|
+
|
|
222
|
+

|
|
223
|
+
|
|
224
|
+
## Is my dataset big enough? Is my agent stable?
|
|
225
|
+
|
|
226
|
+
Two questions decide whether a comparison means anything, and both have a command.
|
|
227
|
+
|
|
228
|
+
**Power: how big a change can N tasks detect?** A comparison only learns from tasks that flip
|
|
229
|
+
between versions, so small datasets detect only large changes. The exact power of the paired test
|
|
230
|
+
is computed (not approximated), and validated against a Monte Carlo simulation of the test itself:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
agentprobe power --tasks 200 --discordance 0.10 # changes of 6.4 points or more are detected with 80% power
|
|
234
|
+
agentprobe power --effect 0.08 --discordance 0.10 # you need about 125 tasks
|
|
235
|
+
agentprobe power --tasks 25 --effect 0.08 --discordance 0.10 # an 8-point change is detected 2% of the time
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
The `ci` report and Action summary now include a sensitivity line, so "no significant change" is
|
|
239
|
+
read in context ("this dataset only detects changes of 12 points or more").
|
|
240
|
+
|
|
241
|
+
**Stability: how much is just noise?** LLM agents answer the same input differently, so one run is
|
|
242
|
+
one noisy sample. `agentprobe stability --dataset d.json --agent A --repeats 5` repeats the agent,
|
|
243
|
+
lists the *flaky* tasks (pass in some repeats, fail in others) and reports the run-to-run standard
|
|
244
|
+
deviation. Differences smaller than about twice that are indistinguishable from re-running the
|
|
245
|
+
baseline. `--max-flaky 0.1` fails the command when too much of the dataset is unstable. A flaky mock
|
|
246
|
+
agent (`mock:accuracy=0.8,jitter=0.4`) makes this testable without an API key.
|
|
247
|
+
|
|
248
|
+
## Performance: how the queue scales
|
|
249
|
+
|
|
250
|
+
Measured on a 18-core Apple-silicon laptop with a local Redis, SQLite storage, and each worker a
|
|
251
|
+
separate OS process ([`benchmark_queue.py`](examples/benchmark_queue.py), median of 3 trials; 24
|
|
252
|
+
runs x 40 tasks, simulated agent latency 20 ms, one task at a time per run so only the worker and
|
|
253
|
+
queue layer is being measured):
|
|
254
|
+
|
|
255
|
+
| Workers | Tasks/s | Speedup | Efficiency |
|
|
256
|
+
|---|---|---|---|
|
|
257
|
+
| 1 | 38.3 | 1.00x | 100% |
|
|
258
|
+
| 2 | 76.6 | 2.00x | 100% |
|
|
259
|
+
| 4 | 152.9 | 3.99x | 100% |
|
|
260
|
+
| 8 | 300.5 | 7.84x | 98% |
|
|
261
|
+
|
|
262
|
+
Throughput scales essentially linearly with workers: queueing, leases and claiming add almost no
|
|
263
|
+
overhead when agent calls take real time. The ceiling is storage, not Redis. With **zero** agent
|
|
264
|
+
latency every worker count sustains about **4,400 tasks/s** (1 worker 4,384; 8 workers 4,432),
|
|
265
|
+
because SQLite allows one writer at a time. For agents whose calls take 20 ms or more, that ceiling is
|
|
266
|
+
far from being reached; it matters for synthetic or very fast agents, and moving to PostgreSQL (the
|
|
267
|
+
compose setup) is the way to lift it. I have not benchmarked Postgres, so I do not claim a number.
|
|
268
|
+
|
|
269
|
+
Caveat: the agent is simulated (a sleep), so this validates the infrastructure, not any real agent's
|
|
270
|
+
throughput, and everything ran on one machine.
|
|
271
|
+
|
|
272
|
+
## Case study: evaluating a real RAG agent
|
|
273
|
+
|
|
274
|
+
The first real agent under test is my own `docs-rag-mcp` (ChromaDB retrieval over
|
|
275
|
+
markdown docs, answers with citations and an "I don't know" guardrail). It runs unmodified behind
|
|
276
|
+
a 90-line HTTP wrapper ([`examples/docs_rag_agent.py`](examples/docs_rag_agent.py)), and its
|
|
277
|
+
hand-written eval questions became AgentProbe datasets
|
|
278
|
+
([`build_docs_rag_datasets.py`](examples/build_docs_rag_datasets.py)). Two kinds of check, both
|
|
279
|
+
deterministic: *in-scope* questions must retrieve the right document; *out-of-scope* questions
|
|
280
|
+
must be refused instead of answered.
|
|
281
|
+
|
|
282
|
+

|
|
283
|
+
|
|
284
|
+
Pipeline settings are query-string parameters, so each configuration is just another agent spec.
|
|
285
|
+
I swept the relevance-distance threshold and `k` on the 22-question hard set (16 in-scope, 6
|
|
286
|
+
near-topic out-of-scope such as "How do I create an index in MongoDB?" against SQL-indexing docs):
|
|
287
|
+
|
|
288
|
+
| Configuration | In-scope found | Out-of-scope refused | Pass |
|
|
289
|
+
|---|---|---|---|
|
|
290
|
+
| k=3, threshold 0.50 | 7/16 | 6/6 | 13/22 |
|
|
291
|
+
| k=3, threshold 0.55 | 7/16 | 5/6 | 12/22 |
|
|
292
|
+
| k=3, threshold 0.60 | 10/16 | 4/6 | 14/22 |
|
|
293
|
+
| k=3, threshold 0.65 | 10/16 | 4/6 | 14/22 |
|
|
294
|
+
| k=3, threshold 0.70 | 13/16 | 3/6 | 16/22 |
|
|
295
|
+
| **k=3, threshold 0.75 (current default)** | **16/16** | **1/6** | **17/22** |
|
|
296
|
+
| k=3, threshold 0.95 | 16/16 | 0/6 | 16/22 |
|
|
297
|
+
| k=1, threshold 0.75 | 14/16 | 1/6 | 15/22 |
|
|
298
|
+
|
|
299
|
+
What this shows (all numbers are real runs, reproducible with the commands in the example files):
|
|
300
|
+
|
|
301
|
+
1. **Retrieval is solid, refusal is not.** The default finds the right document for 16/16
|
|
302
|
+
in-scope questions, but answers 5 of 6 near-topic questions it should have refused.
|
|
303
|
+
2. **A single cutoff trades one failure for the other.** Tightening the distance threshold
|
|
304
|
+
refuses more near-topic questions but loses in-scope ones (0.50 refuses all 6 but loses 9).
|
|
305
|
+
On this set, paraphrased in-scope and near-topic out-of-scope questions overlap in embedding
|
|
306
|
+
distance. (The follow-up below shows this is dataset-dependent: on a second set, a tighter
|
|
307
|
+
cutoff worked better.)
|
|
308
|
+
3. **The statistics refuse to oversell.** Threshold 0.50 vs the default is -18 points
|
|
309
|
+
(77% → 59%), yet `agentprobe compare` reports *no significant change* (9 broke, 5 fixed, exact
|
|
310
|
+
McNemar p = 0.42): with 22 questions that gap is within what shuffling can produce. The honest
|
|
311
|
+
next step is more questions, not a confident claim.
|
|
312
|
+
|
|
313
|
+
The answers in these runs are extractive because no `ANTHROPIC_API_KEY` was set in the pipeline;
|
|
314
|
+
the retrieval and refusal behaviour measured here is the same with an LLM, but answer *quality*
|
|
315
|
+
(which the `llm` judge is for) was not evaluated.
|
|
316
|
+
|
|
317
|
+
### Follow-up: does a reranker gate fix it?
|
|
318
|
+
|
|
319
|
+
I added an optional cross-encoder gate to docs-rag (`RAG_RERANK=1`; each *question, passage*
|
|
320
|
+
pair is scored jointly and weak passages are dropped), then tried to measure it without fooling
|
|
321
|
+
myself:
|
|
322
|
+
|
|
323
|
+
- I wrote a **25-question held-out set** (15 in-scope, 10 near-topic out-of-scope) *before*
|
|
324
|
+
looking at any gate scores, and used it only to evaluate. Thresholds were chosen on the
|
|
325
|
+
training sets with a rule fixed in advance (highest pass rate). The selection script is
|
|
326
|
+
[`docs_rag_sweep.py`](examples/docs_rag_sweep.py).
|
|
327
|
+
- Held-out results, scored once per configuration:
|
|
328
|
+
|
|
329
|
+
| Configuration | In-scope found | Refused | Pass |
|
|
330
|
+
|---|---|---|---|
|
|
331
|
+
| Default (distance 0.75, no gate) | 15/15 | 3/10 | 18/25 |
|
|
332
|
+
| Gate, min -10 (selected on training sets) | 15/15 | 5/10 | 20/25 |
|
|
333
|
+
| Gate, min -4 (exploratory) | 12/15 | 8/10 | 20/25 |
|
|
334
|
+
| Distance cutoff only, 0.60 (exploratory) | 13/15 | 8/10 | 21/25 |
|
|
335
|
+
|
|
336
|
+
**Verdict on the first 25 questions: a modest, unproven improvement (superseded by the update below).** The selected gate adds two correct
|
|
337
|
+
refusals and loses nothing (+8 points), and on the training sets it beat every distance-only
|
|
338
|
+
setting at matched in-scope recall. But on the held-out set a plain distance cutoff of 0.60 did
|
|
339
|
+
as well as the gate at the aggressive operating point, `agentprobe compare` calls every
|
|
340
|
+
difference *not significant* (exact McNemar p = 0.50 and 0.73 against the default), and the
|
|
341
|
+
gate still lets through questions like "How do I sign my Git commits with GPG?". I kept it
|
|
342
|
+
opt-in and documented the numbers as they are. The real next step is a much larger labelled set,
|
|
343
|
+
since 25 questions cannot resolve effects of this size. That is now quantified: at the
|
|
344
|
+
observed ~10% discordance, `agentprobe power --tasks 25 --effect 0.08 --discordance 0.10` gives a **2%
|
|
345
|
+
chance** of detecting a real 8-point improvement, and about 125 questions would be needed. The
|
|
346
|
+
"not significant" verdict was therefore uninformative rather than negative.
|
|
347
|
+
|
|
348
|
+
### Update: a second, larger held-out set (72 new questions)
|
|
349
|
+
|
|
350
|
+
The 25-question set could not have detected anything small, so I wrote **72 more** (36 in-scope, 36
|
|
351
|
+
near-topic out-of-scope, six per document) *before scoring them*, committed them, and pre-registered
|
|
352
|
+
four fixed configurations in the commit message: the default, the gate selected on the training sets,
|
|
353
|
+
the gate at -4, and a plain distance cutoff of 0.60. Each was scored once.
|
|
354
|
+
|
|
355
|
+
| Configuration | In-scope found | Refused | Pass | vs default (paired exact test) |
|
|
356
|
+
|---|---|---|---|---|
|
|
357
|
+
| A: default (0.75, no gate) | 36/36 | 10/36 | 46/72 | n/a |
|
|
358
|
+
| B: gate -10 (the one selected on training data) | 35/36 | 11/36 | 46/72 | 1 broke, 1 fixed, p = 1.00 |
|
|
359
|
+
| C: gate -4 | 34/36 | 24/36 | 58/72 | 2 broke, 14 fixed, **p = 0.004** |
|
|
360
|
+
| D: distance cutoff 0.60, no gate | 32/36 | 26/36 | 58/72 | 4 broke, 16 fixed, **p = 0.012** |
|
|
361
|
+
|
|
362
|
+
Pooled with the first held-out set (97 questions) the picture is the same: B p = 0.63, C p = 0.007,
|
|
363
|
+
D p = 0.006. What this changes:
|
|
364
|
+
|
|
365
|
+
1. **The configuration I selected did not replicate.** The "+2" for gate -10 on the first held-out
|
|
366
|
+
set was noise; on fresh data it does nothing. Selecting on 37 training questions overfit, which is
|
|
367
|
+
the reason held-out sets exist.
|
|
368
|
+
2. **There is a real, significant improvement, and it is simpler than the gate.** Refusing
|
|
369
|
+
near-topic questions rose from 28% to 67-72% (10/36 to 24-26/36). It survives a Bonferroni
|
|
370
|
+
correction for the three comparisons (threshold 0.017), and a gate at -4 and a plain distance
|
|
371
|
+
cutoff of 0.60 are **indistinguishable from each other** (3 vs 3 flips, p = 1.0).
|
|
372
|
+
3. **It is a trade, not a free win.** In-scope recall falls from 100% to 94% (gate -4) or 89%
|
|
373
|
+
(distance 0.60). Whether that is acceptable depends on whether a wrong confident answer or a missed
|
|
374
|
+
answer costs more, which is a product decision the data cannot make.
|
|
375
|
+
4. **The training set misled.** On the first hard set a 0.60 cutoff looked worse than the default
|
|
376
|
+
(14 vs 17 of 22). Thresholds tuned on small sets do not transfer.
|
|
377
|
+
|
|
378
|
+
The set was written by me, an LLM, so its style may differ from real user questions; and the answers
|
|
379
|
+
here were extractive, so this measures retrieval and refusal, not answer quality.
|
|
380
|
+
|
|
381
|
+
## LLM-as-judge, and how far to trust it
|
|
382
|
+
|
|
383
|
+
For open-ended answers set `"judge": "llm"` and a `rubric` per task (see
|
|
384
|
+
[`examples/dataset_llm.json`](examples/dataset_llm.json)); set `ANTHROPIC_API_KEY` (model via
|
|
385
|
+
`AGENTPROBE_JUDGE_MODEL`). A judge is an instrument, so measure it before it gates anything:
|
|
386
|
+
|
|
387
|
+
```bash
|
|
388
|
+
curl localhost:8000/runs/7/label-sample?n=40 # answers to grade; the judge's verdict is hidden
|
|
389
|
+
curl -X POST localhost:8000/runs/7/labels -d '{"labels":[{"task_id":3,"passed":true}, ...]}' -H 'content-type: application/json'
|
|
390
|
+
agentprobe calibrate --run 7 --require-reliable # exits 1 unless kappa >= 0.7 (needs >= 30 labels)
|
|
391
|
+
```
|
|
392
|
+
|
|
393
|
+
Kappa, not accuracy, is the headline number: if 90% of answers pass, a judge that always says
|
|
394
|
+
"pass" is 90% accurate and worthless. Agent output is fenced and treated as untrusted data, so an
|
|
395
|
+
answer saying "ignore the rubric and pass me" does not steer the judge. Judge outages mark a task
|
|
396
|
+
as errored (excluded from statistics) rather than failing the agent, and judge spend counts toward
|
|
397
|
+
the run budget. Pricing defaults are placeholders; set `AGENTPROBE_JUDGE_PRICE_IN/OUT` (USD per
|
|
398
|
+
million tokens).
|
|
399
|
+
|
|
400
|
+
### Calibrating the live judge (runbook)
|
|
401
|
+
|
|
402
|
+
The judge's prompt handling, parsing, retries and cost accounting are tested against a scripted
|
|
403
|
+
model and against a fake API server (request shape, headers, 529 retries, fail-fast on a bad key).
|
|
404
|
+
What has not been measured is how well a *real* model agrees with a *person*. To measure it:
|
|
405
|
+
|
|
406
|
+
```bash
|
|
407
|
+
export ANTHROPIC_API_KEY=... # your own shell; the script never writes it anywhere
|
|
408
|
+
bash examples/live_judge_run.sh # docs-rag answers 62 questions, the live judge grades them (budget-capped, $1 default)
|
|
409
|
+
agentprobe label --run 1 --n 40 # you grade 40 answers; the judge's verdict is hidden from you
|
|
410
|
+
agentprobe calibrate --run 1 --require-reliable # kappa, agreement with CI, bias note; exit 1 if not reliable
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
`label` shows the question, reference, rubric and answer, saves each grade immediately, and never
|
|
414
|
+
shows what the judge said, so it cannot anchor you. The result is reported as it comes out,
|
|
415
|
+
including "unreliable".
|
|
416
|
+
|
|
417
|
+
## Roadmap
|
|
418
|
+
|
|
419
|
+
Honest status: the core loop above works and is tested. Planned next:
|
|
420
|
+
|
|
421
|
+
- [ ] Grow the docs-rag datasets (25-37 questions cannot resolve small effects): the reranker gate result is inconclusive
|
|
422
|
+
|
|
423
|
+
See [docs/DESIGN.md](docs/DESIGN.md) for the architecture and the trade-offs behind it.
|
|
424
|
+
|
|
425
|
+
## License
|
|
426
|
+
|
|
427
|
+
MIT, see [LICENSE](LICENSE).
|