evalcrucible 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agents.py +92 -0
- audit.py +67 -0
- evalcrucible-0.1.0.dist-info/METADATA +132 -0
- evalcrucible-0.1.0.dist-info/RECORD +12 -0
- evalcrucible-0.1.0.dist-info/WHEEL +5 -0
- evalcrucible-0.1.0.dist-info/entry_points.txt +2 -0
- evalcrucible-0.1.0.dist-info/licenses/LICENSE +21 -0
- evalcrucible-0.1.0.dist-info/top_level.txt +6 -0
- experiment.py +126 -0
- graders.py +94 -0
- run_eval.py +35 -0
- suite.py +203 -0
agents.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The agents under test. All deterministic, all documented — the lab
|
|
2
|
+
measures the *suite and the graders*, so the agents are instruments,
|
|
3
|
+
not the object of study.
|
|
4
|
+
|
|
5
|
+
- reference_agent: correct by construction on the clean suite. One
|
|
6
|
+
deliberate exception: for tasks marked `shared_table`, it reads its
|
|
7
|
+
answer from SHARED_TABLE — the same source the gold labels were
|
|
8
|
+
generated from. When that source is wrong, agent and gold are wrong
|
|
9
|
+
together, and no output-based check can see it. That is the point.
|
|
10
|
+
- degraded_agent: a plausible weaker agent with three systematic
|
|
11
|
+
errors (no random noise — systematic errors are what evals exist
|
|
12
|
+
to catch):
|
|
13
|
+
* multiplication: computes a*(b-1) when b is greater than 5;
|
|
14
|
+
* extraction: on total/amount questions it returns the day of
|
|
15
|
+
the month from the date in the text instead of the money;
|
|
16
|
+
* classification: a lexicon with no negation handling.
|
|
17
|
+
- null_agent: returns the empty string. Scores 0 on any honest suite.
|
|
18
|
+
- echo_agent: returns the prompt verbatim.
|
|
19
|
+
"""
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
# The shared source: gold labels for shared_table tasks were generated
|
|
23
|
+
# from this table, and the reference agent reads it too. Its flaw-09
|
|
24
|
+
# entry is wrong (the task text says 315) — by design.
|
|
25
|
+
SHARED_TABLE = {"flaw-09": "300"}
|
|
26
|
+
|
|
27
|
+
_POS = {"gorgeous", "superb", "impressive", "impressed", "excellent",
|
|
28
|
+
"great", "wonderful", "best", "faster", "lighter", "worked",
|
|
29
|
+
"resolved", "fast", "quiet"}
|
|
30
|
+
_NEG = {"broke", "damaged", "crashes", "confusing", "drains",
|
|
31
|
+
"disappointment", "loose", "flickers", "crackle", "frustrating",
|
|
32
|
+
"stopped", "late", "bad", "expensive", "poor"}
|
|
33
|
+
|
|
34
|
+
_FIELD_PHRASE = {"total": "Total due:", "amount": "Amount paid:",
|
|
35
|
+
"fee": "Fee charged:", "rate": "Room rate:",
|
|
36
|
+
"charge": "Delivery charge:", "cost": "Renewal cost:"}
|
|
37
|
+
|
|
38
|
+
def _classify(text, use_negation):
|
|
39
|
+
toks = re.findall(r"[a-z']+", text.lower())
|
|
40
|
+
score = 0
|
|
41
|
+
for i, tok in enumerate(toks):
|
|
42
|
+
val = 1 if tok in _POS else -1 if tok in _NEG else 0
|
|
43
|
+
if val and use_negation and i > 0 and toks[i - 1] == "not":
|
|
44
|
+
val = -val
|
|
45
|
+
score += val
|
|
46
|
+
return "positive" if score > 0 else "negative"
|
|
47
|
+
|
|
48
|
+
def _extract(task, date_confusion=False):
|
|
49
|
+
if date_confusion and task["field"] in ("total", "amount"):
|
|
50
|
+
m = re.search(r"\d{4}-\d{2}-(\d{2})", task["prompt"])
|
|
51
|
+
if m:
|
|
52
|
+
return str(int(m.group(1)))
|
|
53
|
+
phrase = _FIELD_PHRASE[task["field"]]
|
|
54
|
+
m = re.search(re.escape(phrase) + r"\s*(\d+)", task["prompt"])
|
|
55
|
+
return m.group(1) if m else ""
|
|
56
|
+
|
|
57
|
+
def _arith(task, degraded=False):
|
|
58
|
+
a, b, op = task["a"], task["b"], task["op"]
|
|
59
|
+
if op == "add":
|
|
60
|
+
return str(a + b)
|
|
61
|
+
if op == "sub":
|
|
62
|
+
return str(a - b)
|
|
63
|
+
if degraded and b > 5:
|
|
64
|
+
return str(a * (b - 1))
|
|
65
|
+
return str(a * b)
|
|
66
|
+
|
|
67
|
+
def reference_agent(task):
|
|
68
|
+
if task["family"] == "arith":
|
|
69
|
+
return _arith(task)
|
|
70
|
+
if task["family"] == "extract":
|
|
71
|
+
# The reference pipeline reads shared-source tables where they
|
|
72
|
+
# exist — the same source the gold labels came from.
|
|
73
|
+
if task.get("shared_table"):
|
|
74
|
+
return SHARED_TABLE[task["id"]]
|
|
75
|
+
return _extract(task)
|
|
76
|
+
return _classify(task["text"], use_negation=True)
|
|
77
|
+
|
|
78
|
+
def degraded_agent(task):
|
|
79
|
+
if task["family"] == "arith":
|
|
80
|
+
return _arith(task, degraded=True)
|
|
81
|
+
if task["family"] == "extract":
|
|
82
|
+
return _extract(task, date_confusion=True)
|
|
83
|
+
return _classify(task["text"], use_negation=False)
|
|
84
|
+
|
|
85
|
+
def null_agent(task):
|
|
86
|
+
return ""
|
|
87
|
+
|
|
88
|
+
def echo_agent(task):
|
|
89
|
+
return task["prompt"]
|
|
90
|
+
|
|
91
|
+
AGENTS = {"reference": reference_agent, "degraded": degraded_agent,
|
|
92
|
+
"null": null_agent, "echo": echo_agent}
|
audit.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""The benchmark audit: four checks that need no knowledge of which
|
|
2
|
+
tests are broken. Each check flags tasks with a stated reason; the
|
|
3
|
+
audit's worth is measured against planted ground truth it never sees.
|
|
4
|
+
|
|
5
|
+
A. reference-failure — a task a known-correct agent fails is a
|
|
6
|
+
suspect test, not a suspect agent.
|
|
7
|
+
B. probe-pass — a task that a null, echo or constant agent passes
|
|
8
|
+
is measuring nothing.
|
|
9
|
+
C. mutation — if a deliberately wrong answer (off by one, times
|
|
10
|
+
ten, flipped label, swapped entity) still passes, the checker
|
|
11
|
+
is too loose to grade with.
|
|
12
|
+
D. duplication — an eval task whose prompt duplicates a dev task
|
|
13
|
+
(or another eval task) is leakage: the score partly measures
|
|
14
|
+
memorization of the tuning split.
|
|
15
|
+
"""
|
|
16
|
+
import re
|
|
17
|
+
from agents import reference_agent, null_agent, echo_agent
|
|
18
|
+
from graders import passes
|
|
19
|
+
|
|
20
|
+
def _norm_prompt(p):
|
|
21
|
+
return re.sub(r"\s+", " ", p.strip().lower())
|
|
22
|
+
|
|
23
|
+
def _tokens(s):
|
|
24
|
+
return set(re.findall(r"[a-z0-9]+", s.lower()))
|
|
25
|
+
|
|
26
|
+
def _mutants(task, out):
|
|
27
|
+
"""Deliberately-wrong variants of a correct output."""
|
|
28
|
+
import graders
|
|
29
|
+
muts = []
|
|
30
|
+
n = graders._first_number(out)
|
|
31
|
+
if n is not None:
|
|
32
|
+
muts.append(str(int(n) + 1) if n == int(n) else str(n + 1))
|
|
33
|
+
muts.append(str(int(n) * 10) if n == int(n) else str(n * 10))
|
|
34
|
+
if out.strip() == "positive":
|
|
35
|
+
muts.append("negative")
|
|
36
|
+
elif out.strip() == "negative":
|
|
37
|
+
muts.append("positive")
|
|
38
|
+
nums = re.findall(r"\d+", task["prompt"])
|
|
39
|
+
if nums and out.strip() != nums[0]:
|
|
40
|
+
muts.append(nums[0])
|
|
41
|
+
return muts
|
|
42
|
+
|
|
43
|
+
def audit_suite(tasks, dev_tasks=()):
|
|
44
|
+
flags = {} # task_id -> list of reasons
|
|
45
|
+
def flag(tid, reason):
|
|
46
|
+
flags.setdefault(tid, []).append(reason)
|
|
47
|
+
for t in tasks:
|
|
48
|
+
ref_out = reference_agent(t)
|
|
49
|
+
if not passes(ref_out, t):
|
|
50
|
+
flag(t["id"], "A: reference agent fails this test")
|
|
51
|
+
for name, probe in (("null", null_agent), ("echo", echo_agent)):
|
|
52
|
+
if passes(probe(t), t):
|
|
53
|
+
flag(t["id"], f"B: {name} agent passes this test")
|
|
54
|
+
if passes("0", t):
|
|
55
|
+
flag(t["id"], "B: constant agent passes this test")
|
|
56
|
+
if any(passes(m, t) for m in _mutants(t, ref_out)):
|
|
57
|
+
flag(t["id"], "C: a deliberately wrong answer passes")
|
|
58
|
+
dev_norm = {_norm_prompt(d["prompt"]): d["id"] for d in dev_tasks}
|
|
59
|
+
seen = {}
|
|
60
|
+
for t in tasks:
|
|
61
|
+
np_ = _norm_prompt(t["prompt"])
|
|
62
|
+
if np_ in dev_norm:
|
|
63
|
+
flag(t["id"], f"D: prompt duplicates dev task {dev_norm[np_]}")
|
|
64
|
+
if np_ in seen:
|
|
65
|
+
flag(t["id"], f"D: prompt duplicates eval task {seen[np_]}")
|
|
66
|
+
seen[np_] = t["id"]
|
|
67
|
+
return flags
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evalcrucible
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Prove your graders: a grader-validation and benchmark-audit toolkit for agent builders — validate graders against labelled data and audit static prompt-to-gold suites for broken tests.
|
|
5
|
+
Author: Ranadhir Ghosh
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ranadhir19/evalcrucible
|
|
8
|
+
Keywords: agent-evaluation,llm-evals,grader-validation,benchmark-audit,ai-agents,evaluation
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# evalcrucible
|
|
21
|
+
|
|
22
|
+
**Prove your graders: a grader-validation and benchmark-audit toolkit for agent builders.**
|
|
23
|
+
|
|
24
|
+
Companion lab to The Architect's AI Workshop, Agent Series Ep. 4 —
|
|
25
|
+
"Prove It Works: Building an Eval That Can Catch a Broken Agent
|
|
26
|
+
(and a Broken Benchmark)".
|
|
27
|
+
|
|
28
|
+
Most agent content teaches you to build the agent. This lab teaches
|
|
29
|
+
you to audit the measuring instrument: validate your graders against
|
|
30
|
+
labelled data, and run an audit that catches broken tests in a static
|
|
31
|
+
(prompt → gold) task suite — because a benchmark with flawed tests
|
|
32
|
+
measures the tests. It does not evaluate agent runs; to score how an
|
|
33
|
+
agent executed a recorded run, see runreckon
|
|
34
|
+
(github.com/ranadhir19/runreckon). (The
|
|
35
|
+
field's object lesson: OpenAI stopped reporting SWE-bench Verified in
|
|
36
|
+
February 2026 after an audit found 59.4% of the reviewed problems had
|
|
37
|
+
flawed tests.)
|
|
38
|
+
|
|
39
|
+
Pure Python standard library. No dependencies, no API keys, fully
|
|
40
|
+
deterministic.
|
|
41
|
+
|
|
42
|
+
## What's in the box
|
|
43
|
+
|
|
44
|
+
| File | What it does |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `suite.py` | A clean 60-task suite (arithmetic, extraction, classification) with gold answers cross-checked by an independent second implementation — and a flawed twin: the same suite with 12 tests replaced by planted broken versions. |
|
|
47
|
+
| `agents.py` | Four instruments: a reference agent (correct by construction), a degraded agent (three documented systematic errors), a null agent (returns nothing), an echo agent (returns the prompt). |
|
|
48
|
+
| `graders.py` | Task checkers, plus three graders (exact, normalized, rubric) whose accuracy is itself measured. The rubric grader is a deterministic stand-in for a cheap model judge; the procedure being taught — validate any grader against labels — is grader-agnostic. |
|
|
49
|
+
| `audit.py` | The audit: four checks that need no knowledge of which tests are broken. A: a known-correct agent failing a test makes the test the suspect. B: a test that null/echo/constant agents pass measures nothing. C: a test that deliberately wrong answers pass is too loose. D: an eval task duplicating a dev task is leakage. |
|
|
50
|
+
| `run_eval.py` | Scores the four instrument agents on both suites, with Wilson 95% intervals — the audit's raw material, not a run-evaluation product. |
|
|
51
|
+
| `experiment.py` | The pre-registered experiment; writes `RESULTS.md`. |
|
|
52
|
+
| `PREREGISTRATION.md` | The hypotheses and acceptance bounds, frozen before the first run. |
|
|
53
|
+
| `validate.py` | The harness: compile, unit tests, and an experiment re-run that must reproduce `RESULTS.md` byte for byte. |
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
|
|
57
|
+
The lab has no dependencies beyond Python itself — every file is
|
|
58
|
+
pure standard library, and there are no API keys, config files or
|
|
59
|
+
environment variables to set. Developed and tested with Python 3.12
|
|
60
|
+
(any current Python 3 works).
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git clone https://github.com/ranadhir19/evalcrucible.git
|
|
64
|
+
cd evalcrucible
|
|
65
|
+
python3 validate.py # self-check: must end with OVERALL PASS
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
That is the whole install: nothing to build, no package to install.
|
|
69
|
+
The harness compiles every file, runs the unit tests, and re-runs
|
|
70
|
+
the experiment, requiring `RESULTS.md` to come back byte-identical —
|
|
71
|
+
so a final line of `OVERALL PASS` means the lab on your machine
|
|
72
|
+
reproduces the published numbers exactly.
|
|
73
|
+
|
|
74
|
+
## Run it
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
python3 validate.py # harness: everything below, checked
|
|
78
|
+
python3 run_eval.py # agent scores on both suites
|
|
79
|
+
python3 experiment.py # full experiment; rewrites RESULTS.md
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
There is no server to configure and nothing takes command-line
|
|
83
|
+
arguments: the three scripts are the whole interface. `run_eval.py`
|
|
84
|
+
prints every agent's pass count and rate with Wilson 95% intervals
|
|
85
|
+
for the clean and flawed suites; `experiment.py` prints the full
|
|
86
|
+
pre-registered experiment and writes the same text to `RESULTS.md`.
|
|
87
|
+
|
|
88
|
+
## What the experiment found (measured; see RESULTS.md)
|
|
89
|
+
|
|
90
|
+
- The reference agent scores **100%** on the clean suite and **91.7%**
|
|
91
|
+
on the flawed suite. It did not get worse — five of its "failures"
|
|
92
|
+
are test bugs (wrong gold, swapped labels, an impossible format).
|
|
93
|
+
- The null agent scores **0** on the clean suite and **2/60** on the
|
|
94
|
+
flawed one. The echo agent — it returns the question as the answer —
|
|
95
|
+
scores **4/60**. A broken benchmark pays agents that do nothing.
|
|
96
|
+
- The audit detects **11 of the 12** planted flaws and raises **zero**
|
|
97
|
+
false flags on the clean suite. The one it misses is the
|
|
98
|
+
shared-source flaw: the gold label and the reference solver were
|
|
99
|
+
both generated from the same wrong table, so every output-based
|
|
100
|
+
check passes. That class is only catchable by independently
|
|
101
|
+
re-deriving the gold — which is why the clean suite's answers are
|
|
102
|
+
cross-checked by a second implementation here.
|
|
103
|
+
- Grader validation on 150 labelled items: the **exact** grader
|
|
104
|
+
agrees with gold labels only **80.0%** of the time (kappa 0.545) —
|
|
105
|
+
it rejects correct answers for their formatting, and it *fails* the
|
|
106
|
+
pre-registered acceptance bound. The boring **normalized** grader
|
|
107
|
+
scores 100%. The cheap **rubric** grader scores **99.3%**
|
|
108
|
+
(kappa 0.986) and is *accepted* under the frozen bound — with one
|
|
109
|
+
documented miss: it passed a wrong extraction answer ("54", the
|
|
110
|
+
shipment number from the prompt) because the number was close and
|
|
111
|
+
appeared in the prompt. A judge that has been validated has a
|
|
112
|
+
measured error rate; a judge that hasn't is a hope.
|
|
113
|
+
|
|
114
|
+
## The honest criteria
|
|
115
|
+
|
|
116
|
+
The bounds in `PREREGISTRATION.md` were written before the experiment
|
|
117
|
+
ran and they do not move. Had the rubric grader missed its bound, the
|
|
118
|
+
lab would have fallen back to the strictest grader that cleared it,
|
|
119
|
+
and said so. Had the audit detected fewer than 10 of 12, that would
|
|
120
|
+
be the headline. Report the measurement, not the hope.
|
|
121
|
+
|
|
122
|
+
## Method notes and limits
|
|
123
|
+
|
|
124
|
+
- The agents are instruments, not products: the degraded agent's
|
|
125
|
+
errors are systematic and documented so score movements are
|
|
126
|
+
attributable.
|
|
127
|
+
- At 60 tasks the Wilson intervals are wide (roughly ±10 points).
|
|
128
|
+
That is itself one of the lab's results: differences smaller than
|
|
129
|
+
the interval are not evidence, for anyone's benchmark.
|
|
130
|
+
- The rubric grader is a deterministic proxy. Re-run the same
|
|
131
|
+
validation procedure with a real model judge before trusting one;
|
|
132
|
+
the procedure, not the proxy, is the deliverable.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
agents.py,sha256=2ofE0KFI4TkXyN6Uc5nR5-AFKSn0AgfRGPbd-3Zmj6Q,3695
|
|
2
|
+
audit.py,sha256=sUaNv58Sxp-6gR400OJ7M9BDa0fF6iZKJsA7UkxPdvc,2672
|
|
3
|
+
experiment.py,sha256=BFc-ywsrYfzfSB90fZaYT5gvQtk68X_Ln-qbyAJNT-I,6079
|
|
4
|
+
graders.py,sha256=BwjGwRbNGxZfnp32xRyA8HTEn1WVggaKsFon4kpXP8A,3679
|
|
5
|
+
run_eval.py,sha256=fRSAb2c-qaE4zPX4D2wDmyzkQASgBPL0QqUHekaqmaY,1243
|
|
6
|
+
suite.py,sha256=rg4oibqXDCH3fWWeuxaUXcuw9vgjjEwM4EXY0SB2d2U,11614
|
|
7
|
+
evalcrucible-0.1.0.dist-info/licenses/LICENSE,sha256=MT0RhdXKVJ3nautcORSCWgZxgrPyQTS6bni3NzQas-I,1071
|
|
8
|
+
evalcrucible-0.1.0.dist-info/METADATA,sha256=8FXtqxf1jvHHzSa9ZyJWX3OSKSzIPqTfnE6MJEzkL8U,7102
|
|
9
|
+
evalcrucible-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
evalcrucible-0.1.0.dist-info/entry_points.txt,sha256=X34OGN40c40YXwMGv-vhjvilRMK3zxg_Azxz1PIpXAA,47
|
|
11
|
+
evalcrucible-0.1.0.dist-info/top_level.txt,sha256=_7DHd1bYrvx5TbOWl1G7WcvgreiVpLYH30iMlBRs5qs,47
|
|
12
|
+
evalcrucible-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ranadhir Ghosh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
experiment.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""The pre-registered experiment (see PREREGISTRATION.md).
|
|
2
|
+
|
|
3
|
+
Builds the labelled subset, validates the three graders against gold
|
|
4
|
+
labels, runs the audit on the clean and flawed suites, scores all
|
|
5
|
+
agents on both suites, and tests H1/H2/H3 against the frozen bounds.
|
|
6
|
+
Writes RESULTS.md with every number, pass or fail.
|
|
7
|
+
"""
|
|
8
|
+
import os
|
|
9
|
+
import suite, agents, graders
|
|
10
|
+
from audit import audit_suite
|
|
11
|
+
from run_eval import score
|
|
12
|
+
|
|
13
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
14
|
+
|
|
15
|
+
# ------------------------------------------------------------ labelled set
|
|
16
|
+
def labelled_subset():
|
|
17
|
+
clean = suite.clean_suite()
|
|
18
|
+
picked = ([t for t in clean if t["family"] == "arith"][:10]
|
|
19
|
+
+ [t for t in clean if t["family"] == "extract"][:10]
|
|
20
|
+
+ [t for t in clean if t["family"] == "classify"][:10])
|
|
21
|
+
items = [] # (task, output, gold_pass)
|
|
22
|
+
for t in picked:
|
|
23
|
+
exp = t["expected"]
|
|
24
|
+
items.append((t, exp, True)) # plain
|
|
25
|
+
if t["family"] == "classify":
|
|
26
|
+
items.append((t, exp.capitalize(), True)) # formatted
|
|
27
|
+
items.append((t, "negative" if exp == "positive" else "positive", False)) # delta
|
|
28
|
+
items.append((t, "neutral", False)) # swap
|
|
29
|
+
else:
|
|
30
|
+
v = int(exp)
|
|
31
|
+
items.append((t, f"{v:,}" if v >= 1000 else f"{v}.00", True)) # formatted
|
|
32
|
+
items.append((t, str(v + 1), False)) # delta
|
|
33
|
+
if t["family"] == "arith":
|
|
34
|
+
a, b, op = t["a"], t["b"], t["op"]
|
|
35
|
+
wrong = {"mul": a + b, "add": a - b, "sub": a + b}[op]
|
|
36
|
+
else:
|
|
37
|
+
import re
|
|
38
|
+
wrong = re.findall(r"\d+", t["prompt"])[0]
|
|
39
|
+
items.append((t, str(wrong), False)) # swap
|
|
40
|
+
items.append((t, "", False)) # empty
|
|
41
|
+
return items
|
|
42
|
+
|
|
43
|
+
def kappa(pairs):
|
|
44
|
+
tp = sum(1 for g, p in pairs if g and p); tn = sum(1 for g, p in pairs if not g and not p)
|
|
45
|
+
fp = sum(1 for g, p in pairs if not g and p); fn = sum(1 for g, p in pairs if g and not p)
|
|
46
|
+
n = len(pairs); po = (tp + tn) / n
|
|
47
|
+
pe = ((tp + fn) * (tp + fp) + (fp + tn) * (fn + tn)) / (n * n)
|
|
48
|
+
return po, (po - pe) / (1 - pe) if pe < 1 else 1.0
|
|
49
|
+
|
|
50
|
+
def main():
|
|
51
|
+
clean, flawed, dev = suite.clean_suite(), suite.flawed_suite(), suite.dev_suite()
|
|
52
|
+
L = []
|
|
53
|
+
def w(s=""):
|
|
54
|
+
L.append(s); print(s)
|
|
55
|
+
|
|
56
|
+
w("# Results — agent-eval-lab experiment")
|
|
57
|
+
w()
|
|
58
|
+
w("All numbers measured by `experiment.py`; bounds frozen in PREREGISTRATION.md.")
|
|
59
|
+
w()
|
|
60
|
+
# ---- agent scores
|
|
61
|
+
w("## Agent scores (Wilson 95% intervals)")
|
|
62
|
+
scores = {}
|
|
63
|
+
for sname, tasks in (("clean", clean), ("flawed", flawed)):
|
|
64
|
+
for aname in agents.AGENTS:
|
|
65
|
+
s = score(aname, tasks); scores[(sname, aname)] = s
|
|
66
|
+
w(f"- {sname} / {aname}: {s['passed']}/{s['total']} = {100*s['rate']:.1f}% "
|
|
67
|
+
f"(95% CI {100*s['lo']:.1f}-{100*s['hi']:.1f})")
|
|
68
|
+
w()
|
|
69
|
+
# ---- audit
|
|
70
|
+
flags_clean = audit_suite(clean, dev)
|
|
71
|
+
flags_flawed = audit_suite(flawed, dev)
|
|
72
|
+
detected = sorted(set(suite.PLANTED_FLAWS) & set(flags_flawed))
|
|
73
|
+
missed = sorted(set(suite.PLANTED_FLAWS) - set(flags_flawed))
|
|
74
|
+
false_flags = sorted(set(flags_clean))
|
|
75
|
+
w("## Audit (checks A-D in audit.py)")
|
|
76
|
+
w(f"- planted flaws: 12; detected: {len(detected)} {detected}")
|
|
77
|
+
w(f"- missed: {len(missed)} {missed}")
|
|
78
|
+
for m in missed:
|
|
79
|
+
w(f" - {m}: {suite.PLANTED_FLAWS[m]}")
|
|
80
|
+
w(f"- flags raised on the clean suite: {len(false_flags)} of 60 {false_flags}")
|
|
81
|
+
for tid in sorted(flags_flawed):
|
|
82
|
+
w(f" - {tid}: {'; '.join(flags_flawed[tid])}")
|
|
83
|
+
w()
|
|
84
|
+
# ---- grader validation
|
|
85
|
+
items = labelled_subset()
|
|
86
|
+
w(f"## Grader validation on the labelled subset ({len(items)} items, "
|
|
87
|
+
f"{sum(1 for _, _, g in items if g)} gold-pass / {sum(1 for _, _, g in items if not g)} gold-fail)")
|
|
88
|
+
gres = {}
|
|
89
|
+
for gname, gfn in graders.GRADERS.items():
|
|
90
|
+
pairs = [(gold, gfn(out, t["expected"], t["prompt"])) for t, out, gold in items]
|
|
91
|
+
agree, kap = kappa(pairs)
|
|
92
|
+
gres[gname] = (agree, kap)
|
|
93
|
+
w(f"- {gname}: agreement {100*agree:.1f}%, Cohen's kappa {kap:.3f}")
|
|
94
|
+
w()
|
|
95
|
+
# ---- hypotheses
|
|
96
|
+
w("## Hypotheses (bounds frozen before the run)")
|
|
97
|
+
h1 = len(detected) >= 10 and len(false_flags) <= 3
|
|
98
|
+
w(f"- H1 audit detection: {'CONFIRMED' if h1 else 'NOT CONFIRMED'} — "
|
|
99
|
+
f"detected {len(detected)}/12 (bound 10), false flags {len(false_flags)}/60 (bound 3)")
|
|
100
|
+
rubric_ok = gres["rubric"][0] >= 0.90 and gres["rubric"][1] >= 0.75
|
|
101
|
+
fallback = None
|
|
102
|
+
if not rubric_ok:
|
|
103
|
+
for gname in ("normalized", "exact"):
|
|
104
|
+
if gres[gname][0] >= 0.90 and gres[gname][1] >= 0.75:
|
|
105
|
+
fallback = gname; break
|
|
106
|
+
h2_line = (f"rubric grader agreement {100*gres['rubric'][0]:.1f}% / kappa {gres['rubric'][1]:.3f} "
|
|
107
|
+
f"vs bound 90% / 0.75 — ")
|
|
108
|
+
if rubric_ok:
|
|
109
|
+
w(f"- H2 grader acceptance: rubric ACCEPTED — " + h2_line + "bound met")
|
|
110
|
+
else:
|
|
111
|
+
w(f"- H2 grader acceptance: rubric REJECTED — " + h2_line +
|
|
112
|
+
(f"bound missed; lab falls back to the {fallback} grader" if fallback
|
|
113
|
+
else "bound missed and no grader clears it"))
|
|
114
|
+
ref_drop = 100 * (scores[("clean", "reference")]["rate"] - scores[("flawed", "reference")]["rate"])
|
|
115
|
+
h3 = scores[("clean", "null")]["passed"] == 0 and scores[("flawed", "null")]["passed"] >= 2 and ref_drop >= 4
|
|
116
|
+
w(f"- H3 false-pass cost: {'CONFIRMED' if h3 else 'NOT CONFIRMED'} — "
|
|
117
|
+
f"null agent 0/{scores[('clean','null')]['total']} on clean, "
|
|
118
|
+
f"{scores[('flawed','null')]['passed']}/{scores[('flawed','null')]['total']} on flawed; "
|
|
119
|
+
f"reference drops {ref_drop:.1f} points on the flawed suite")
|
|
120
|
+
w()
|
|
121
|
+
w(f"Degraded agent: {100*scores[('clean','degraded')]['rate']:.1f}% on clean, "
|
|
122
|
+
f"{100*scores[('flawed','degraded')]['rate']:.1f}% on flawed.")
|
|
123
|
+
open(os.path.join(HERE, "RESULTS.md"), "w").write("\n".join(L) + "\n")
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
main()
|
graders.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Checkers (score one task output against its test) and graders
|
|
2
|
+
(score an output against a gold answer, for grader validation).
|
|
3
|
+
|
|
4
|
+
The distinction matters: a *checker* is part of a benchmark — if it is
|
|
5
|
+
broken, the benchmark is broken. A *grader* is a measurement tool you
|
|
6
|
+
choose — and any grader, cheap or expensive, must itself be validated
|
|
7
|
+
against labelled data before its scores mean anything.
|
|
8
|
+
"""
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
# ---------------------------------------------------------------- checkers
|
|
12
|
+
def _first_number(s):
|
|
13
|
+
m = re.search(r"-?\d[\d,]*\.?\d*", str(s))
|
|
14
|
+
return float(m.group(0).replace(",", "")) if m else None
|
|
15
|
+
|
|
16
|
+
def check_exact(out, task):
|
|
17
|
+
return str(out).strip() == str(task["expected"]).strip()
|
|
18
|
+
|
|
19
|
+
def check_numeric(out, task):
|
|
20
|
+
got, want = _first_number(out), _first_number(task["expected"])
|
|
21
|
+
return got is not None and want is not None and abs(got - want) < 1e-9
|
|
22
|
+
|
|
23
|
+
def check_loose_numeric(out, task):
|
|
24
|
+
got, want = _first_number(out), _first_number(task["expected"])
|
|
25
|
+
return got is not None and want is not None and abs(got - want) <= 0.5 * abs(want)
|
|
26
|
+
|
|
27
|
+
def check_any(out, task):
|
|
28
|
+
return True
|
|
29
|
+
|
|
30
|
+
def check_echo(out, task):
|
|
31
|
+
out = str(out)
|
|
32
|
+
if str(task["expected"]) in out:
|
|
33
|
+
return True
|
|
34
|
+
words = task["prompt"].split()
|
|
35
|
+
return sum(w in out for w in words) >= 0.8 * len(words)
|
|
36
|
+
|
|
37
|
+
def check_contains(out, task):
|
|
38
|
+
return str(task["expected"]) in str(out)
|
|
39
|
+
|
|
40
|
+
def check_has_digit(out, task):
|
|
41
|
+
return any(c.isdigit() for c in str(out))
|
|
42
|
+
|
|
43
|
+
CHECKERS = {"exact": check_exact, "numeric": check_numeric,
|
|
44
|
+
"loose_numeric": check_loose_numeric, "any": check_any,
|
|
45
|
+
"echo": check_echo, "contains": check_contains,
|
|
46
|
+
"has_digit": check_has_digit}
|
|
47
|
+
|
|
48
|
+
def passes(out, task):
|
|
49
|
+
return CHECKERS[task["checker"]](out, task)
|
|
50
|
+
|
|
51
|
+
# ----------------------------------------------------------------- graders
|
|
52
|
+
def grade_exact(out, expected):
|
|
53
|
+
return str(out).strip() == str(expected).strip()
|
|
54
|
+
|
|
55
|
+
def _normalize(s):
|
|
56
|
+
s = str(s).lower().strip()
|
|
57
|
+
s = s.replace(",", "")
|
|
58
|
+
s = re.sub(r"\s+", " ", s)
|
|
59
|
+
s = s.rstrip(".")
|
|
60
|
+
n = _first_number(s)
|
|
61
|
+
if n is not None and re.fullmatch(r"-?\d[\d,]*\.?\d*\s*(dollars)?", str(s).strip().lower().replace(",", "")):
|
|
62
|
+
return f"num:{n:g}"
|
|
63
|
+
return s
|
|
64
|
+
|
|
65
|
+
def grade_normalized(out, expected):
|
|
66
|
+
return _normalize(out) == _normalize(expected)
|
|
67
|
+
|
|
68
|
+
def _tokens(s):
|
|
69
|
+
return set(re.findall(r"[a-z0-9]+", str(s).lower()))
|
|
70
|
+
|
|
71
|
+
def grade_rubric(out, expected, prompt=""):
|
|
72
|
+
"""A deterministic stand-in for a cheap model judge, with its
|
|
73
|
+
formula fixed a priori: similarity to the expected answer, numeric
|
|
74
|
+
closeness, and a 'groundedness' term rewarding outputs whose
|
|
75
|
+
tokens appear in the prompt. Pass at a score of 0.5 or above.
|
|
76
|
+
|
|
77
|
+
It exists to be validated, not to be trusted: experiment.py
|
|
78
|
+
measures its agreement with gold labels against the pre-registered
|
|
79
|
+
acceptance bound in PREREGISTRATION.md."""
|
|
80
|
+
jac = len(_tokens(out) & _tokens(expected)) / max(1, len(_tokens(out) | _tokens(expected)))
|
|
81
|
+
got, want = _first_number(out), _first_number(expected)
|
|
82
|
+
if got is not None and want is not None:
|
|
83
|
+
closeness = 1.0 if got == want else max(0.0, 1.0 - abs(got - want) / max(1.0, abs(want)))
|
|
84
|
+
elif got is None and want is None:
|
|
85
|
+
closeness = 0.5
|
|
86
|
+
else:
|
|
87
|
+
closeness = 0.0
|
|
88
|
+
ptoks = _tokens(prompt)
|
|
89
|
+
echo = (sum(t in ptoks for t in _tokens(out)) / max(1, len(_tokens(out)))) if prompt else 0.0
|
|
90
|
+
return 0.4 * jac + 0.3 * closeness + 0.3 * echo >= 0.5
|
|
91
|
+
|
|
92
|
+
GRADERS = {"exact": lambda out, exp, prompt="": grade_exact(out, exp),
|
|
93
|
+
"normalized": lambda out, exp, prompt="": grade_normalized(out, exp),
|
|
94
|
+
"rubric": grade_rubric}
|
run_eval.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Run agents against suites and report scores with Wilson score
|
|
2
|
+
intervals. A pass rate without an interval is a marketing number:
|
|
3
|
+
at 60 tasks, a 95% interval is roughly ±10 points wide."""
|
|
4
|
+
import math
|
|
5
|
+
from agents import AGENTS
|
|
6
|
+
from graders import passes
|
|
7
|
+
|
|
8
|
+
def wilson(k, n, z=1.96):
|
|
9
|
+
if n == 0:
|
|
10
|
+
return (0.0, 0.0, 0.0)
|
|
11
|
+
p = k / n
|
|
12
|
+
den = 1 + z * z / n
|
|
13
|
+
cen = (p + z * z / (2 * n)) / den
|
|
14
|
+
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
|
|
15
|
+
return p, max(0.0, cen - half), min(1.0, cen + half)
|
|
16
|
+
|
|
17
|
+
def score(agent_name, tasks):
|
|
18
|
+
fn = AGENTS[agent_name]
|
|
19
|
+
k = sum(1 for t in tasks if passes(fn(t), t))
|
|
20
|
+
p, lo, hi = wilson(k, len(tasks))
|
|
21
|
+
return dict(agent=agent_name, passed=k, total=len(tasks),
|
|
22
|
+
rate=p, lo=lo, hi=hi)
|
|
23
|
+
|
|
24
|
+
def main():
|
|
25
|
+
import suite
|
|
26
|
+
for sname, tasks in (("clean", suite.clean_suite()),
|
|
27
|
+
("flawed", suite.flawed_suite())):
|
|
28
|
+
print(f"--- {sname} suite ({len(tasks)} tasks)")
|
|
29
|
+
for aname in AGENTS:
|
|
30
|
+
s = score(aname, tasks)
|
|
31
|
+
print(f"{aname:>10}: {s['passed']:>2}/{s['total']} "
|
|
32
|
+
f"{100*s['rate']:5.1f}% (95% CI {100*s['lo']:.1f}-{100*s['hi']:.1f})")
|
|
33
|
+
|
|
34
|
+
if __name__ == "__main__":
|
|
35
|
+
main()
|
suite.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Task suites for agent-eval-lab.
|
|
2
|
+
|
|
3
|
+
A task is a dict: id, family, prompt, expected, checker, split.
|
|
4
|
+
Checkers are named grading functions in graders.py.
|
|
5
|
+
|
|
6
|
+
The CLEAN suite's expected outputs come from the reference solver in
|
|
7
|
+
agents.py and are cross-checked by `independent_expected()` below — a
|
|
8
|
+
second, deliberately differently-written implementation. validate.py
|
|
9
|
+
asserts the two agree on every clean task.
|
|
10
|
+
|
|
11
|
+
The FLAWED suite is the clean suite with 12 of its tests replaced by
|
|
12
|
+
planted broken versions. PLANTED_FLAWS is the ground-truth manifest
|
|
13
|
+
the audit is scored against; the audit itself never reads it.
|
|
14
|
+
"""
|
|
15
|
+
import random
|
|
16
|
+
|
|
17
|
+
# ---------------------------------------------------------------- clean data
|
|
18
|
+
_ARITH_TEMPLATES = [
|
|
19
|
+
("A crate holds {a} bottles. A shop orders {b} crates. How many bottles is that in total?", "mul"),
|
|
20
|
+
("Sara picked {a} apples and then picked {b} more. How many apples does she have now?", "add"),
|
|
21
|
+
("A tank had {a} litres of water. {b} litres were drained. How many litres remain?", "sub"),
|
|
22
|
+
("{a} workers each assemble {b} parts in a day. How many parts do they assemble in total?", "mul"),
|
|
23
|
+
("Tom had {a} stickers. He bought {b} more packs of stickers. How many stickers does he have now?", "add"),
|
|
24
|
+
("A rope is {a} metres long. A piece of {b} metres is cut off. How many metres are left?", "sub"),
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
def _arith_tasks():
|
|
28
|
+
rng = random.Random(4101)
|
|
29
|
+
tasks = []
|
|
30
|
+
for i in range(24):
|
|
31
|
+
tmpl, op = _ARITH_TEMPLATES[i % len(_ARITH_TEMPLATES)]
|
|
32
|
+
if op == "mul":
|
|
33
|
+
a, b = rng.randint(3, 14), rng.randint(3, 12)
|
|
34
|
+
ans = a * b
|
|
35
|
+
elif op == "add":
|
|
36
|
+
a, b = rng.randint(15, 240), rng.randint(12, 180)
|
|
37
|
+
ans = a + b
|
|
38
|
+
else:
|
|
39
|
+
a, b = rng.randint(60, 300), rng.randint(10, 59)
|
|
40
|
+
ans = a - b
|
|
41
|
+
tasks.append(dict(id=f"arith-{i+1:02d}", family="arith", op=op, a=a, b=b,
|
|
42
|
+
prompt=tmpl.format(a=a, b=b), expected=str(ans),
|
|
43
|
+
checker="numeric", split="eval"))
|
|
44
|
+
return tasks
|
|
45
|
+
|
|
46
|
+
_EXTRACT_TEXTS = [
|
|
47
|
+
("Invoice 114, dated 2026-03-14. Customer: Northwind. Total due: 482 dollars.", "total", "482"),
|
|
48
|
+
("Order 77 shipped on 2026-01-05. Items: 3 cables. Amount paid: 129 dollars.", "amount", "129"),
|
|
49
|
+
("Ticket 9021 opened 2025-11-30. Priority: high. Fee charged: 75 dollars.", "fee", "75"),
|
|
50
|
+
("Booking 331, check-in 2026-06-18. Room rate: 210 dollars per night.", "rate", "210"),
|
|
51
|
+
("Invoice 208, dated 2026-02-01. Customer: Fabrikam. Total due: 967 dollars.", "total", "967"),
|
|
52
|
+
("Shipment 54 arrived 2026-04-22. Weight: 18 kg. Delivery charge: 46 dollars.", "charge", "46"),
|
|
53
|
+
("Order 618 shipped on 2025-12-09. Items: 12 notebooks. Amount paid: 240 dollars.", "amount", "240"),
|
|
54
|
+
("Membership 88 renewed 2026-05-11. Plan: annual. Renewal cost: 320 dollars.", "cost", "320"),
|
|
55
|
+
("Ticket 417 opened 2026-07-03. Priority: low. Fee charged: 25 dollars.", "fee", "25"),
|
|
56
|
+
("Invoice 522, dated 2026-08-19. Customer: Contoso. Total due: 154 dollars.", "total", "154"),
|
|
57
|
+
("Booking 905, check-in 2026-09-27. Room rate: 145 dollars per night.", "rate", "145"),
|
|
58
|
+
("Shipment 301 arrived 2026-10-02. Weight: 9 kg. Delivery charge: 38 dollars.", "charge", "38"),
|
|
59
|
+
("Order 246 shipped on 2026-03-29. Items: 7 lamps. Amount paid: 693 dollars.", "amount", "693"),
|
|
60
|
+
("Membership 19 renewed 2025-10-21. Plan: monthly. Renewal cost: 60 dollars.", "cost", "60"),
|
|
61
|
+
("Invoice 760, dated 2026-11-08. Customer: Adatum. Total due: 728 dollars.", "total", "728"),
|
|
62
|
+
("Ticket 655 opened 2026-01-17. Priority: medium. Fee charged: 90 dollars.", "fee", "90"),
|
|
63
|
+
("Booking 129, check-in 2025-12-25. Room rate: 380 dollars per night.", "rate", "380"),
|
|
64
|
+
("Shipment 812 arrived 2026-05-30. Weight: 27 kg. Delivery charge: 64 dollars.", "charge", "64"),
|
|
65
|
+
]
|
|
66
|
+
_FIELD_WORD = {"total": "total due", "amount": "amount paid", "fee": "fee charged",
|
|
67
|
+
"rate": "room rate", "charge": "delivery charge", "cost": "renewal cost"}
|
|
68
|
+
|
|
69
|
+
def _extract_tasks():
|
|
70
|
+
tasks = []
|
|
71
|
+
for i, (text, field, ans) in enumerate(_EXTRACT_TEXTS):
|
|
72
|
+
tasks.append(dict(id=f"extract-{i+1:02d}", family="extract", field=field,
|
|
73
|
+
prompt=f"{text} What is the {_FIELD_WORD[field]}? Answer with the number only.",
|
|
74
|
+
expected=ans, checker="numeric", split="eval"))
|
|
75
|
+
return tasks
|
|
76
|
+
|
|
77
|
+
_CLASSIFY_ITEMS = [
|
|
78
|
+
("The battery lasts all day and the screen is gorgeous", "positive"),
|
|
79
|
+
("The update broke my saved files and support never replied", "negative"),
|
|
80
|
+
("Setup took two minutes and everything just worked", "positive"),
|
|
81
|
+
("It is not fast, and the fan is not quiet either", "negative"),
|
|
82
|
+
("The camera is superb in low light, truly impressive", "positive"),
|
|
83
|
+
("Delivery was late and the box arrived damaged", "negative"),
|
|
84
|
+
("Not bad at all — the keyboard feels great", "positive"),
|
|
85
|
+
("The app crashes every time I open settings", "negative"),
|
|
86
|
+
("Customer service resolved it within an hour, excellent", "positive"),
|
|
87
|
+
("The instructions were confusing and the parts did not fit", "negative"),
|
|
88
|
+
("It is not expensive, and the quality is not poor", "positive"),
|
|
89
|
+
("Battery drains in an hour, a real disappointment", "negative"),
|
|
90
|
+
("The new model is faster and noticeably lighter", "positive"),
|
|
91
|
+
("Not impressed — the hinge feels loose already", "negative"),
|
|
92
|
+
("A wonderful gift, she uses it every single day", "positive"),
|
|
93
|
+
("The screen flickers and the speakers crackle", "negative"),
|
|
94
|
+
("Honestly the best purchase I have made this year", "positive"),
|
|
95
|
+
("It stopped working after a week, very frustrating", "negative"),
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
def _classify_tasks():
|
|
99
|
+
return [dict(id=f"classify-{i+1:02d}", family="classify",
|
|
100
|
+
prompt=f"Classify the sentiment of this review as positive or negative: \"{text}\" Answer with one word.",
|
|
101
|
+
text=text, expected=label, checker="exact", split="eval")
|
|
102
|
+
for i, (text, label) in enumerate(_CLASSIFY_ITEMS)]
|
|
103
|
+
|
|
104
|
+
def clean_suite():
|
|
105
|
+
return _arith_tasks() + _extract_tasks() + _classify_tasks()
|
|
106
|
+
|
|
107
|
+
# A small dev split, used only to demonstrate leakage detection.
|
|
108
|
+
# Its tasks are generated with a different seed, so no dev prompt
|
|
109
|
+
# coincides with an eval prompt — except the one planted leak.
|
|
110
|
+
def dev_suite():
|
|
111
|
+
rng = random.Random(777)
|
|
112
|
+
dev = []
|
|
113
|
+
for i in range(6):
|
|
114
|
+
tmpl, op = _ARITH_TEMPLATES[i % len(_ARITH_TEMPLATES)]
|
|
115
|
+
if op == "mul":
|
|
116
|
+
a, b = rng.randint(3, 14), rng.randint(3, 12); ans = a * b
|
|
117
|
+
elif op == "add":
|
|
118
|
+
a, b = rng.randint(15, 240), rng.randint(12, 180); ans = a + b
|
|
119
|
+
else:
|
|
120
|
+
a, b = rng.randint(60, 300), rng.randint(10, 59); ans = a - b
|
|
121
|
+
dev.append(dict(id=f"dev-arith-{i+1:02d}", family="arith", op=op, a=a, b=b,
|
|
122
|
+
prompt=tmpl.format(a=a, b=b), expected=str(ans),
|
|
123
|
+
checker="numeric", split="dev"))
|
|
124
|
+
return dev
|
|
125
|
+
|
|
126
|
+
# ------------------------------------------------------------- flawed suite
|
|
127
|
+
# The flawed suite is the clean suite with 12 of its tests replaced by
|
|
128
|
+
# broken versions — the way real benchmarks break: the task stays, the
|
|
129
|
+
# test is wrong. Each flaw is a class from the SWE-bench audit style:
|
|
130
|
+
# wrong gold, degenerate checker, leakage, loose checker, impossible
|
|
131
|
+
# format. PLANTED_FLAWS is the ground-truth manifest the audit is
|
|
132
|
+
# scored against; the audit never reads it.
|
|
133
|
+
def _flawed_replacements():
|
|
134
|
+
arith = {t["id"]: t for t in _arith_tasks()}
|
|
135
|
+
ext = {t["id"]: t for t in _extract_tasks()}
|
|
136
|
+
cls = {t["id"]: t for t in _classify_tasks()}
|
|
137
|
+
rep = {}
|
|
138
|
+
|
|
139
|
+
t = dict(arith["arith-02"]); t.update(id="flaw-01",
|
|
140
|
+
expected=str(int(t["expected"]) + 1)); rep["arith-02"] = t # wrong gold (+1)
|
|
141
|
+
t = dict(ext["extract-03"]); t.update(id="flaw-02", checker="any"); rep["extract-03"] = t # checker accepts everything
|
|
142
|
+
t = dict(cls["classify-05"]); t.update(id="flaw-03", expected=""); rep["classify-05"] = t # only the empty answer passes
|
|
143
|
+
t = dict(ext["extract-06"]); t.update(id="flaw-04", checker="echo"); rep["extract-06"] = t # parroting the prompt passes
|
|
144
|
+
t = dict(arith["arith-01"]); t.update(id="flaw-05", checker="loose_numeric"); rep["arith-01"] = t # ±50% tolerance
|
|
145
|
+
t = dict(dev_suite()[0]); t.update(id="flaw-06", split="eval"); rep["arith-04"] = t # dev task leaked into eval, displacing arith-04
|
|
146
|
+
t = dict(arith["arith-05"]); t.update(id="flaw-07", checker="exact",
|
|
147
|
+
expected=f" {int(t['expected']):,} dollars "); rep["arith-05"] = t # format no correct answer can match
|
|
148
|
+
t = dict(ext["extract-01"]); t.update(id="flaw-08", checker="contains"); rep["extract-01"] = t # substring: "482" also matches "4820"
|
|
149
|
+
# flaw-09: the gold comes from the same lookup the reference solver
|
|
150
|
+
# uses, and the lookup is wrong — a shared-source error. The task
|
|
151
|
+
# text says the amount is 315; the shared table (and gold) say 300.
|
|
152
|
+
t = dict(ext["extract-02"])
|
|
153
|
+
t.update(id="flaw-09", shared_table=True,
|
|
154
|
+
prompt="Order 77 shipped on 2026-01-05. Items: 3 cables. Amount paid: 315 dollars. "
|
|
155
|
+
"What is the amount paid? Answer with the number only.",
|
|
156
|
+
expected="300"); rep["extract-02"] = t
|
|
157
|
+
a = dict(cls["classify-01"]); a.update(id="flaw-10", expected="negative"); rep["classify-01"] = a # swapped gold labels
|
|
158
|
+
b = dict(cls["classify-02"]); b.update(id="flaw-11", expected="positive"); rep["classify-02"] = b
|
|
159
|
+
t = dict(arith["arith-03"]); t.update(id="flaw-12", checker="has_digit"); rep["arith-03"] = t # any answer containing a digit passes
|
|
160
|
+
return rep
|
|
161
|
+
|
|
162
|
+
PLANTED_FLAWS = {
|
|
163
|
+
"flaw-01": "wrong gold value",
|
|
164
|
+
"flaw-02": "checker accepts any output",
|
|
165
|
+
"flaw-03": "only the empty answer passes",
|
|
166
|
+
"flaw-04": "echoing the prompt passes",
|
|
167
|
+
"flaw-05": "numeric tolerance of ±50 percent",
|
|
168
|
+
"flaw-06": "dev task duplicated into the eval split",
|
|
169
|
+
"flaw-07": "expected format no correct answer can match",
|
|
170
|
+
"flaw-08": "substring checker (482 also matches 4820)",
|
|
171
|
+
"flaw-09": "gold and solver share one wrong source table",
|
|
172
|
+
"flaw-10": "swapped gold label (pair, 1 of 2)",
|
|
173
|
+
"flaw-11": "swapped gold label (pair, 2 of 2)",
|
|
174
|
+
"flaw-12": "any answer containing a digit passes",
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
def flawed_suite():
|
|
178
|
+
rep = _flawed_replacements()
|
|
179
|
+
return [rep.get(t["id"], t) for t in clean_suite()]
|
|
180
|
+
|
|
181
|
+
# --------------------------------------- independent cross-check of the gold
|
|
182
|
+
def independent_expected(task):
|
|
183
|
+
"""Second implementation of every clean task's expected output,
|
|
184
|
+
written differently from agents.reference_agent on purpose."""
|
|
185
|
+
import re
|
|
186
|
+
if task["family"] == "arith":
|
|
187
|
+
a, b, op = task["a"], task["b"], task["op"]
|
|
188
|
+
return str({"add": a + b, "sub": a - b, "mul": a * b}[op])
|
|
189
|
+
if task["family"] == "extract":
|
|
190
|
+
m = re.search(r"(\d+) dollars", task["prompt"])
|
|
191
|
+
return m.group(1) if m else None
|
|
192
|
+
if task["family"] == "classify":
|
|
193
|
+
neg = ("not ", "never", "broke", "damaged", "crashes", "confusing",
|
|
194
|
+
"drains", "disappointment", "loose", "flickers", "crackle",
|
|
195
|
+
"frustrating", "stopped working", "late and")
|
|
196
|
+
pos = ("gorgeous", "just worked", "superb", "great", "excellent",
|
|
197
|
+
"impressive", "wonderful", "best purchase", "faster and noticeably")
|
|
198
|
+
text = task["text"].lower()
|
|
199
|
+
if "not bad" in text or "not expensive" in text:
|
|
200
|
+
return "positive"
|
|
201
|
+
score = sum(w in text for w in pos) - sum(w in text for w in neg)
|
|
202
|
+
return "positive" if score > 0 else "negative"
|
|
203
|
+
return None
|