proofbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proofbench/__init__.py +2 -0
- proofbench/capture.py +31 -0
- proofbench/cli.py +41 -0
- proofbench/config.py +68 -0
- proofbench/engine.py +63 -0
- proofbench/judge.py +20 -0
- proofbench/optimize.py +58 -0
- proofbench/runner.py +25 -0
- proofbench/util.py +49 -0
- proofbench-0.1.0.dist-info/METADATA +105 -0
- proofbench-0.1.0.dist-info/RECORD +14 -0
- proofbench-0.1.0.dist-info/WHEEL +4 -0
- proofbench-0.1.0.dist-info/entry_points.txt +2 -0
- proofbench-0.1.0.dist-info/licenses/LICENSE +21 -0
proofbench/__init__.py
ADDED
proofbench/capture.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""git-diff capture: reset a repo to its broken state, then snapshot what the agent changed.
|
|
2
|
+
|
|
3
|
+
The dominant agent-eval case is "agent edits a repo, grade the diff against an answer
|
|
4
|
+
branch." Dependency and cache trees are excluded: they are never the agent's fix and can
|
|
5
|
+
balloon the judge prompt past ARG_MAX."""
|
|
6
|
+
import subprocess
|
|
7
|
+
|
|
8
|
+
_EXCLUDES = [":(exclude,glob)**/node_modules/**", ":(exclude,glob)**/.venv/**",
|
|
9
|
+
":(exclude,glob)**/__pycache__/**", ":(exclude,glob)**/.code-review-graph/**"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _git(repo, *args):
|
|
13
|
+
return subprocess.run(["git", "-C", str(repo), *args],
|
|
14
|
+
capture_output=True, text=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def reset(repo, default_branch):
|
|
18
|
+
"""Restore the repo to a pristine broken state. --detach tolerates the branch being
|
|
19
|
+
checked out in another worktree."""
|
|
20
|
+
_git(repo, "checkout", "-q", "--detach", default_branch)
|
|
21
|
+
_git(repo, "reset", "-q", "--hard")
|
|
22
|
+
_git(repo, "clean", "-qfdx")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def ground_truth(repo, default_branch, answer_branch):
|
|
26
|
+
return _git(repo, "diff", f"{default_branch}..{answer_branch}").stdout
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def capture(repo):
|
|
30
|
+
_git(repo, "add", "-N", ".")
|
|
31
|
+
return _git(repo, "diff", "--", ".", *_EXCLUDES).stdout
|
proofbench/cli.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""proofbench {run|optimize} — config-driven eval and self-improvement for agent skills."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from . import config, engine, optimize
|
|
7
|
+
from .util import HarnessError, log
|
|
8
|
+
|
|
9
|
+
DEMO = Path(__file__).resolve().parent.parent / "examples" / "proofbench.demo.yaml"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _config_path(args):
|
|
13
|
+
if args.demo:
|
|
14
|
+
return DEMO if DEMO.is_file() else sys.exit("--demo needs a source checkout; pass --config")
|
|
15
|
+
return args.config or sys.exit("pass --config PATH (or --demo)")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def cmd_run(cfg, args):
|
|
19
|
+
board = engine.bench(cfg, Path("runs") / args.run_id, cfg.models)
|
|
20
|
+
for model, leg in board.items():
|
|
21
|
+
macro = {k: round(v, 3) for k, v in leg.items() if k.startswith("macro_")}
|
|
22
|
+
log(f"[{model}] {macro}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def cmd_optimize(cfg, args):
|
|
26
|
+
optimize.optimize(cfg, Path("runs") / args.run_id)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main():
|
|
30
|
+
parser = argparse.ArgumentParser(prog="proofbench")
|
|
31
|
+
parser.add_argument("command", choices=["run", "optimize"])
|
|
32
|
+
parser.add_argument("--config")
|
|
33
|
+
parser.add_argument("--demo", action="store_true")
|
|
34
|
+
parser.add_argument("--run-id", default="run")
|
|
35
|
+
args = parser.parse_args()
|
|
36
|
+
try:
|
|
37
|
+
cfg = config.load(_config_path(args))
|
|
38
|
+
(cmd_run if args.command == "run" else cmd_optimize)(cfg, args)
|
|
39
|
+
except HarnessError as err:
|
|
40
|
+
print(f"\033[31m[proofbench] {err}\033[0m", file=sys.stderr)
|
|
41
|
+
sys.exit(1)
|
proofbench/config.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Config is the entire domain surface. The engine reads only this object."""
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
from .util import die
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Task:
|
|
12
|
+
name: str
|
|
13
|
+
path: Path # the repo the agent works in
|
|
14
|
+
invoke: str # what the agent is asked to do, e.g. "/crg-debug"
|
|
15
|
+
default_branch: str # the broken / initial state
|
|
16
|
+
answer_branch: str # the ground-truth fixed state
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Config:
|
|
21
|
+
root: Path # config file's directory; all paths resolve against it
|
|
22
|
+
runner: str
|
|
23
|
+
models: list # the subject-driver sweep; the weak leg is the signal
|
|
24
|
+
judge_model: str # held constant — never let the weak model grade itself
|
|
25
|
+
synth_model: str
|
|
26
|
+
objective: str
|
|
27
|
+
concurrency: int
|
|
28
|
+
max_turns: int
|
|
29
|
+
judge_prompt: str
|
|
30
|
+
corpus: list = field(default_factory=list)
|
|
31
|
+
subject: Path = None # the artifact under optimization; None => pure-eval mode
|
|
32
|
+
synth_prompt: str = None # presence (with subject) unlocks `optimize`
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def can_optimize(self):
|
|
36
|
+
return self.subject is not None and self.synth_prompt is not None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load(config_path):
|
|
40
|
+
path = Path(config_path).expanduser().resolve()
|
|
41
|
+
if not path.is_file():
|
|
42
|
+
die(f"config not found: {path}")
|
|
43
|
+
root = path.parent
|
|
44
|
+
raw = yaml.safe_load(path.read_text()) or {}
|
|
45
|
+
|
|
46
|
+
def resolve(value):
|
|
47
|
+
return (root / Path(value).expanduser()).resolve() if value else None
|
|
48
|
+
|
|
49
|
+
corpus = raw.get("corpus") or die("config has no corpus")
|
|
50
|
+
tasks = [Task(name=t["name"], path=resolve(t["path"]), invoke=t["invoke"],
|
|
51
|
+
default_branch=t["default_branch"], answer_branch=t["answer_branch"])
|
|
52
|
+
for t in corpus]
|
|
53
|
+
|
|
54
|
+
models = raw.get("models") or die("config has no models")
|
|
55
|
+
return Config(
|
|
56
|
+
root=root,
|
|
57
|
+
runner=raw.get("runner", "claude"),
|
|
58
|
+
models=models,
|
|
59
|
+
judge_model=raw.get("judge_model", models[0]),
|
|
60
|
+
synth_model=raw.get("synth_model", models[0]),
|
|
61
|
+
objective=raw.get("objective", "macro_recall"),
|
|
62
|
+
concurrency=int(raw.get("concurrency", 3)),
|
|
63
|
+
max_turns=int(raw.get("max_turns", 200)),
|
|
64
|
+
judge_prompt=(resolve(raw.get("judge")) or die("config has no judge prompt")).read_text(),
|
|
65
|
+
corpus=tasks,
|
|
66
|
+
subject=resolve(raw.get("subject")),
|
|
67
|
+
synth_prompt=resolve(raw.get("synth")).read_text() if raw.get("synth") else None,
|
|
68
|
+
)
|
proofbench/engine.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""The bench: corpus -> reset -> run -> capture -> judge -> scoreboard.
|
|
2
|
+
|
|
3
|
+
Models run sequentially (each reuses the same repos); tasks within a model run in
|
|
4
|
+
parallel (each task is a distinct repo, so no working-tree contention)."""
|
|
5
|
+
import json
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
7
|
+
|
|
8
|
+
from . import capture, judge, runner
|
|
9
|
+
from .util import die, log
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _ground_truth(cfg, run_dir):
|
|
13
|
+
gt = {}
|
|
14
|
+
for task in cfg.corpus:
|
|
15
|
+
d = run_dir / "_ground-truth" / task.name
|
|
16
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
17
|
+
answer = capture.ground_truth(task.path, task.default_branch, task.answer_branch)
|
|
18
|
+
readme = next((p for p in task.path.glob("README*")), None)
|
|
19
|
+
(d / "answer.diff").write_text(answer)
|
|
20
|
+
gt[task.name] = (readme.read_text() if readme else "", answer)
|
|
21
|
+
return gt
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _run_and_grade(cfg, task, model, gt, out_dir):
|
|
25
|
+
readme, answer = gt[task.name]
|
|
26
|
+
capture.reset(task.path, task.default_branch)
|
|
27
|
+
runner.run(cfg.runner, task.invoke, model, cfg.max_turns, task.path, out_dir / "run.log")
|
|
28
|
+
agent_diff = capture.capture(task.path)
|
|
29
|
+
(out_dir / "agent.diff").write_text(agent_diff)
|
|
30
|
+
grade = judge.grade(cfg.judge_prompt, readme, answer, agent_diff, cfg.judge_model)
|
|
31
|
+
grade["task"] = task.name
|
|
32
|
+
(out_dir / "grade.json").write_text(json.dumps(grade, indent=2))
|
|
33
|
+
return grade
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def aggregate(grades):
|
|
37
|
+
shared = set.intersection(*(set(judge.numeric_fields(g)) for g in grades))
|
|
38
|
+
macro = {f"macro_{f}": sum(g[f] for g in grades) / len(grades) for f in shared}
|
|
39
|
+
return {**macro, "tasks": {g["task"]: judge.numeric_fields(g) for g in grades}}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def bench(cfg, run_dir, models):
|
|
43
|
+
gt = _ground_truth(cfg, run_dir)
|
|
44
|
+
board = {}
|
|
45
|
+
for model in models:
|
|
46
|
+
log(f"leg [{model}] — {len(cfg.corpus)} tasks, concurrency {cfg.concurrency}")
|
|
47
|
+
with ThreadPoolExecutor(max_workers=cfg.concurrency) as pool:
|
|
48
|
+
def work(task, model=model):
|
|
49
|
+
out = run_dir / model / task.name
|
|
50
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
return _run_and_grade(cfg, task, model, gt, out)
|
|
52
|
+
grades = list(pool.map(work, cfg.corpus))
|
|
53
|
+
board[model] = aggregate(grades)
|
|
54
|
+
(run_dir / model).mkdir(parents=True, exist_ok=True)
|
|
55
|
+
(run_dir / model / "scoreboard.json").write_text(json.dumps(board[model], indent=2))
|
|
56
|
+
(run_dir / "scoreboard.json").write_text(json.dumps(board, indent=2))
|
|
57
|
+
return board
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def objective_of(board_leg, objective):
|
|
61
|
+
if objective not in board_leg:
|
|
62
|
+
die(f"objective '{objective}' not in scoreboard (have {list(board_leg)})")
|
|
63
|
+
return board_leg[objective]
|
proofbench/judge.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""LLM judge: compare the agent's artifact to ground truth and emit numeric metrics.
|
|
2
|
+
|
|
3
|
+
A grade with no numeric metric is a hard stop, not a zero. Failing loud here is the
|
|
4
|
+
whole point: an empty grade that flows onward manufactures false confidence."""
|
|
5
|
+
from .util import claude_json, die
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def numeric_fields(grade):
|
|
9
|
+
return {k: v for k, v in grade.items() if isinstance(v, (int, float)) and not isinstance(v, bool)}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def grade(judge_prompt, readme, answer_diff, agent_diff, model):
|
|
13
|
+
prompt = (f"{judge_prompt}\n\n"
|
|
14
|
+
f"===== README =====\n{readme}\n\n"
|
|
15
|
+
f"===== ANSWER_DIFF =====\n{answer_diff}\n\n"
|
|
16
|
+
f"===== AGENT_DIFF =====\n{agent_diff}\n")
|
|
17
|
+
result = claude_json(prompt, model)
|
|
18
|
+
if not numeric_fields(result):
|
|
19
|
+
die("judge returned no numeric metric")
|
|
20
|
+
return result
|
proofbench/optimize.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""The self-improving loop: synth an improved subject, keep it only if the objective holds.
|
|
2
|
+
|
|
3
|
+
The keep/revert gate is deterministic and harness-owned. The model proposes; the shell's
|
|
4
|
+
comparison decides. A claim the model cannot make is a claim it cannot fake."""
|
|
5
|
+
import json
|
|
6
|
+
import shutil
|
|
7
|
+
|
|
8
|
+
from . import engine
|
|
9
|
+
from .util import claude_json, die, log
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _collect_grades(run_dir, model):
|
|
13
|
+
blocks = []
|
|
14
|
+
for task_dir in sorted((run_dir / model).iterdir()):
|
|
15
|
+
grade = task_dir / "grade.json"
|
|
16
|
+
if grade.is_file():
|
|
17
|
+
blocks.append(f"--- {task_dir.name} ---\n{grade.read_text()}")
|
|
18
|
+
return "\n".join(blocks)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _synth(cfg, run_dir, model):
|
|
22
|
+
prompt = (f"{cfg.synth_prompt}\n\n"
|
|
23
|
+
f"===== GRADES =====\n{_collect_grades(run_dir, model)}\n\n"
|
|
24
|
+
f"===== CURRENT_SUBJECT =====\n{cfg.subject.read_text()}\n")
|
|
25
|
+
result = claude_json(prompt, cfg.synth_model)
|
|
26
|
+
candidate = result.get("subject") or die("synth produced no subject")
|
|
27
|
+
(run_dir / "subject.candidate").write_text(candidate)
|
|
28
|
+
(run_dir / "changelog.md").write_text(result.get("changelog", ""))
|
|
29
|
+
return candidate
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def optimize(cfg, run_dir):
|
|
33
|
+
if not cfg.can_optimize:
|
|
34
|
+
die("optimize needs both `subject` and `synth` in config")
|
|
35
|
+
model = cfg.models[-1] # the weak leg is the optimization target — only misses teach
|
|
36
|
+
log(f"baseline on weak leg [{model}]")
|
|
37
|
+
baseline = engine.bench(cfg, run_dir / "baseline", [model])[model]
|
|
38
|
+
base_score = engine.objective_of(baseline, cfg.objective)
|
|
39
|
+
|
|
40
|
+
log("synthesizing an improved subject from the misses")
|
|
41
|
+
candidate = _synth(cfg, run_dir, model)
|
|
42
|
+
backup = run_dir / "subject.bak"
|
|
43
|
+
shutil.copy(cfg.subject, backup)
|
|
44
|
+
cfg.subject.write_text(candidate)
|
|
45
|
+
|
|
46
|
+
log("re-running with the candidate subject")
|
|
47
|
+
improved = engine.bench(cfg, run_dir / "improved", [model])[model]
|
|
48
|
+
imp_score = engine.objective_of(improved, cfg.objective)
|
|
49
|
+
|
|
50
|
+
kept = imp_score >= base_score
|
|
51
|
+
if not kept:
|
|
52
|
+
shutil.copy(backup, cfg.subject)
|
|
53
|
+
verdict = "KEPT" if kept else "REVERTED"
|
|
54
|
+
log(f"{cfg.objective}: baseline={base_score:.3f} improved={imp_score:.3f} -> {verdict}")
|
|
55
|
+
(run_dir / "result.json").write_text(json.dumps(
|
|
56
|
+
{"objective": cfg.objective, "baseline": base_score,
|
|
57
|
+
"improved": imp_score, "kept": kept}, indent=2))
|
|
58
|
+
return kept
|
proofbench/runner.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Runner adapters: how the subject is driven headless. Driver-agnostic by design.
|
|
2
|
+
|
|
3
|
+
A built-in name (``claude``) maps to a command builder. Any other value is treated as
|
|
4
|
+
a path to an executable invoked as ``<runner> <invoke> <model>`` — bring your own agent."""
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _claude(invoke, model, max_turns):
|
|
9
|
+
return ["claude", "-p", invoke, "--model", model,
|
|
10
|
+
"--dangerously-skip-permissions", "--output-format", "json",
|
|
11
|
+
"--max-turns", str(max_turns)]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
BUILTINS = {"claude": _claude}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run(runner, invoke, model, max_turns, cwd, log_path):
|
|
18
|
+
"""Execute one headless attempt in ``cwd``; capture all output to ``log_path``.
|
|
19
|
+
|
|
20
|
+
The agent's exit status is advisory — what gets graded is the artifact it leaves
|
|
21
|
+
behind (see capture), never the agent's own claim of success."""
|
|
22
|
+
argv = BUILTINS[runner](invoke, model, max_turns) if runner in BUILTINS \
|
|
23
|
+
else [runner, invoke, model]
|
|
24
|
+
with open(log_path, "w") as log:
|
|
25
|
+
subprocess.run(argv, cwd=cwd, stdout=log, stderr=subprocess.STDOUT, text=True)
|
proofbench/util.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Fail-loud helpers. The measurement system must crash on bad data, never average around it."""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class HarnessError(RuntimeError):
|
|
9
|
+
"""A measurement failed. Always fatal — never swallowed into a plausible number."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def die(message):
|
|
13
|
+
raise HarnessError(message)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def log(message):
|
|
17
|
+
print(f"\033[36m[proofbench]\033[0m {message}", file=sys.stderr)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def extract_json(envelope):
|
|
21
|
+
"""Parse a `claude --output-format json` envelope's `.result`, tolerant of a prose
|
|
22
|
+
preamble or markdown fences by slicing the first '{' to the last '}'. Fail loud at
|
|
23
|
+
every step so a narrating model can never yield an empty grade that flows onward."""
|
|
24
|
+
if not envelope.strip():
|
|
25
|
+
die("LLM produced no output")
|
|
26
|
+
try:
|
|
27
|
+
result = json.loads(envelope)["result"]
|
|
28
|
+
except (json.JSONDecodeError, KeyError, TypeError):
|
|
29
|
+
die("LLM envelope was not valid `--output-format json`")
|
|
30
|
+
match = re.search(r"\{.*\}", result, re.DOTALL)
|
|
31
|
+
if not match:
|
|
32
|
+
die("no JSON object found in LLM result")
|
|
33
|
+
try:
|
|
34
|
+
return json.loads(match.group(0))
|
|
35
|
+
except json.JSONDecodeError:
|
|
36
|
+
die("LLM result was not valid JSON")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def claude_json(prompt, model):
|
|
40
|
+
"""Call claude headless and return the parsed JSON object it produced.
|
|
41
|
+
|
|
42
|
+
The prompt goes on STDIN, never argv: an embedded diff can exceed ARG_MAX and
|
|
43
|
+
silently abort exec, which once manufactured a confident wrong number."""
|
|
44
|
+
proc = subprocess.run(
|
|
45
|
+
["claude", "-p", "--model", model,
|
|
46
|
+
"--dangerously-skip-permissions", "--output-format", "json"],
|
|
47
|
+
input=prompt, capture_output=True, text=True,
|
|
48
|
+
)
|
|
49
|
+
return extract_json(proc.stdout)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: proofbench
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Config-driven eval harness that grades and self-improves headless agent skills against ground-truth corpora
|
|
5
|
+
Project-URL: Homepage, https://github.com/CodeBlackwell/proofbench
|
|
6
|
+
Author: LeChristopher Blackwell
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: agent,benchmark,claude,eval,llm,prompt-optimization
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: pyyaml>=6.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# proofbench
|
|
15
|
+
|
|
16
|
+
A config-driven eval harness that grades and self-improves headless agent skills against
|
|
17
|
+
ground-truth corpora. You plant tasks you already have the answers to, run an agent skill
|
|
18
|
+
across them, and grade what it changed against the known answer with an LLM judge. Then you
|
|
19
|
+
let the harness rewrite the skill from its own misses and keep the rewrite only if the score
|
|
20
|
+
holds.
|
|
21
|
+
|
|
22
|
+
It started as the harness behind [crg-debug](examples/crg-debug/), a graph-driven debugging
|
|
23
|
+
skill. The methodology generalizes to any skill whose output you can compare to a known answer.
|
|
24
|
+
|
|
25
|
+
## Why it exists
|
|
26
|
+
|
|
27
|
+
A skill is only as trustworthy as the proof that it works. Anyone can write a prompt that
|
|
28
|
+
sounds like a methodology. The honest way to know is to measure it against ground truth, on
|
|
29
|
+
the model weak enough to embarrass it. Two ideas carry the whole design, both learned the hard
|
|
30
|
+
way (see [METHODOLOGY.md](METHODOLOGY.md)):
|
|
31
|
+
|
|
32
|
+
1. **Fail loud or do not measure.** Every empty or non-numeric grade is a hard stop. An eval
|
|
33
|
+
that fails open does not just miss data, it manufactures false confidence.
|
|
34
|
+
2. **The weak model is the signal, not the noise.** A self-improving loop learns only from
|
|
35
|
+
misses. Frontier models on easy tasks miss nothing, so they teach nothing. The weak leg is
|
|
36
|
+
the curriculum.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
uv tool install proofbench # or: uvx proofbench
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Requires the `claude` CLI on PATH for the default runner and for the judge.
|
|
45
|
+
|
|
46
|
+
## Quickstart (demo)
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/CodeBlackwell/proofbench && cd proofbench
|
|
50
|
+
bash examples/sample-corpus/build.sh # builds two toy repos with buggy + fixed branches
|
|
51
|
+
uvx --from . proofbench run --demo # eval an agent over them, graded vs the answers
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## How it works
|
|
55
|
+
|
|
56
|
+
One YAML config declares everything domain-specific; the engine is generic. The mode is
|
|
57
|
+
implied by what the config contains:
|
|
58
|
+
|
|
59
|
+
- a corpus + runner + judge gives you a **bench** (`run`)
|
|
60
|
+
- adding a `subject` + `synth` unlocks the **self-improving loop** (`optimize`)
|
|
61
|
+
|
|
62
|
+
```yaml
|
|
63
|
+
subject: ~/.claude/skills/crg-debug/SKILL.md # optional; omit for pure-eval mode
|
|
64
|
+
runner: claude # default adapter; any executable works
|
|
65
|
+
models: [opus, sonnet, haiku] # the driver sweep; the weak leg is the signal
|
|
66
|
+
judge_model: opus # held constant; never let a model grade itself
|
|
67
|
+
objective: macro_recall # the metric the keep/revert gate reads
|
|
68
|
+
judge: prompts/judge.md
|
|
69
|
+
synth: prompts/synth.md # present => `optimize` is available
|
|
70
|
+
corpus:
|
|
71
|
+
- name: primes
|
|
72
|
+
path: examples/sample-corpus/repos/primes
|
|
73
|
+
invoke: "Find and fix the bug in this repository."
|
|
74
|
+
default_branch: buggy
|
|
75
|
+
answer_branch: fixed
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
proofbench run --config proofbench.yaml # eval + scoreboard
|
|
80
|
+
proofbench optimize --config proofbench.yaml # baseline -> synth -> re-run -> keep|revert
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### The keep/revert gate
|
|
84
|
+
|
|
85
|
+
`optimize` runs a baseline on the weak leg, asks an LLM to rewrite the subject from the graded
|
|
86
|
+
misses, re-runs, and keeps the rewrite only if the objective did not regress. The decision is
|
|
87
|
+
the harness comparing two numbers, never the model's own claim of success.
|
|
88
|
+
|
|
89
|
+
## Adapters
|
|
90
|
+
|
|
91
|
+
- **Runner** (`runner:`): `claude` is built in. Any other value is an executable invoked as
|
|
92
|
+
`<runner> <invoke> <model>`, so you can drive aider, codex, or a custom agent.
|
|
93
|
+
- **Capture**: the git-diff default resets a repo to its broken branch and snapshots what the
|
|
94
|
+
agent changed, excluding dependency and cache trees so they never pollute or balloon the
|
|
95
|
+
judge prompt.
|
|
96
|
+
|
|
97
|
+
## Bring your own corpus
|
|
98
|
+
|
|
99
|
+
The bundled corpus is two MIT toy repos for the demo. Point `corpus:` at your own repos
|
|
100
|
+
(each with a broken default branch and a fixed answer branch). See [CORPUS.md](CORPUS.md) for
|
|
101
|
+
how to curate tasks that actually discriminate and do not leak their answers to the judge.
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
MIT.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
proofbench/__init__.py,sha256=Fy3yuDLQ3Vl85TIZGNT0Nkv7jP00DgIDO2aYVkwMukw,115
|
|
2
|
+
proofbench/capture.py,sha256=wu7xui016HW7MwuExSBZ6vNn-pJMAqwUJ3T67ni8mKA,1187
|
|
3
|
+
proofbench/cli.py,sha256=Y5-ogVVLGpOz0F0hG4OCbmJm0s4upryGZYCccErBJF8,1412
|
|
4
|
+
proofbench/config.py,sha256=2ijJATqydGKqonH4odJrudittdFjy-wbrtTDo878TrM,2504
|
|
5
|
+
proofbench/engine.py,sha256=C_SIF9AHl3q35YtLsO74lrKD-BiD0IHFI33jfxX3QnM,2634
|
|
6
|
+
proofbench/judge.py,sha256=V-Hvr4--wOFkSpwae_okoyVJzEuBJ-v4foWdBzHEk5w,828
|
|
7
|
+
proofbench/optimize.py,sha256=UX-k-ZkK_73i2t5G5jv4Z5Ni_-DxplOuskANWEUNm9g,2346
|
|
8
|
+
proofbench/runner.py,sha256=Qq2h8I8vLyeUCXTDW0imWDrUavF7MtIsrRGINE16EGs,1066
|
|
9
|
+
proofbench/util.py,sha256=bMDcPrRkIQVNEWpOezT01Shoa1Wq2Q6uZcw66vammgE,1710
|
|
10
|
+
proofbench-0.1.0.dist-info/METADATA,sha256=6YpF3XHUE3IJ4PhvBExqJoE-8zVi4VtHGAA6L8kdjL8,4401
|
|
11
|
+
proofbench-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
12
|
+
proofbench-0.1.0.dist-info/entry_points.txt,sha256=gB7qemTqy5a2nOtMGZveLfZ6XcvL_xRl_CPBDI2h6eg,51
|
|
13
|
+
proofbench-0.1.0.dist-info/licenses/LICENSE,sha256=e8kfO5zUv5eLlK6PcN_JqglUZlelNebEMUsmt4t9g88,1080
|
|
14
|
+
proofbench-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LeChristopher Blackwell
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|