proofbench 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proofbench-0.1.0 → proofbench-0.2.0}/.gitignore +1 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/PKG-INFO +1 -1
- proofbench-0.2.0/corpora/fetch.sh +23 -0
- proofbench-0.2.0/corpora/manifest.yaml +73 -0
- proofbench-0.2.0/corpora/proofbench.yaml +27 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/capture.py +13 -2
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/cli.py +4 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/engine.py +36 -7
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/runner.py +20 -3
- {proofbench-0.1.0 → proofbench-0.2.0}/pyproject.toml +1 -1
- {proofbench-0.1.0 → proofbench-0.2.0}/tests/test_engine.py +2 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/.github/workflows/release.yml +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/CORPUS.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/LICENSE +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/METHODOLOGY.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/README.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/docs/crg-eval-journey.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/docs/crg-eval-results.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/examples/crg-debug/README.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/examples/proofbench.demo.yaml +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/examples/sample-corpus/build.sh +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/prompts/judge.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/prompts/synth.md +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/__init__.py +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/config.py +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/judge.py +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/optimize.py +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/proofbench/util.py +0 -0
- {proofbench-0.1.0 → proofbench-0.2.0}/tests/test_failloud.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proofbench
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Config-driven eval harness that grades and self-improves headless agent skills against ground-truth corpora
|
|
5
5
|
Project-URL: Homepage, https://github.com/CodeBlackwell/proofbench
|
|
6
6
|
Author: LeChristopher Blackwell
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Clone every `ready` entry in manifest.yaml into ./repos, materializing a local
|
|
3
|
+
# default branch (at the pinned commit) and answer branch, with the broken default
|
|
4
|
+
# checked out. Re-runnable. Cloned repos are gitignored; only the manifest is tracked.
|
|
5
|
+
set -euo pipefail
|
|
6
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
7
|
+
cd "$HERE"
|
|
8
|
+
mkdir -p repos
|
|
9
|
+
|
|
10
|
+
python3 - <<'PY' | while IFS='|' read -r name url def ans pin; do
|
|
11
|
+
import yaml
|
|
12
|
+
for e in yaml.safe_load(open("manifest.yaml"))["ready"]:
|
|
13
|
+
print(f"{e['name']}|{e['url']}|{e['default_branch']}|{e['answer_branch']}|{e['pin']}")
|
|
14
|
+
PY
|
|
15
|
+
repo="repos/$name"
|
|
16
|
+
[ -d "$repo/.git" ] || git clone -q "$url" "$repo"
|
|
17
|
+
git -C "$repo" fetch -q origin
|
|
18
|
+
git -C "$repo" checkout -q --detach
|
|
19
|
+
git -C "$repo" branch -f "$def" "$pin"
|
|
20
|
+
git -C "$repo" branch -f "$ans" "origin/$ans"
|
|
21
|
+
git -C "$repo" checkout -q "$def"
|
|
22
|
+
echo "fetched $name @ ${pin:0:7} (default: $def, answer: $ans)"
|
|
23
|
+
done
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# proofbench corpus manifest.
|
|
2
|
+
#
|
|
3
|
+
# This is the corpus the crg-debug evaluation used: 7 tasks total, 4 easy
|
|
4
|
+
# debugging-interview repos plus 3 Tier-1 hard cases. `ready` entries are public repos
|
|
5
|
+
# pinned to an exact commit; `bash fetch.sh` clones each into ./repos (gitignored) at its
|
|
6
|
+
# pin and materializes the broken default branch + the answer branch.
|
|
7
|
+
#
|
|
8
|
+
# Pins are recorded so a fetch is reproducible: the same manifest yields the same repos.
|
|
9
|
+
|
|
10
|
+
# The 4 easy repos, scored as the main sweep.
|
|
11
|
+
ready:
|
|
12
|
+
- name: debugging-challenge
|
|
13
|
+
url: https://github.com/josh-tobin/debugging-challenge.git
|
|
14
|
+
default_branch: master
|
|
15
|
+
answer_branch: working
|
|
16
|
+
pin: 21177bda401cd0128aefbce1b80e1bed98676ad7
|
|
17
|
+
|
|
18
|
+
- name: BugFix
|
|
19
|
+
url: https://github.com/IIIT-Delhi/BugFix.git
|
|
20
|
+
default_branch: main
|
|
21
|
+
answer_branch: solution
|
|
22
|
+
pin: 924895ea7dd687b409222c0ce91eec7f3c883f78
|
|
23
|
+
|
|
24
|
+
- name: ph2-raccoon-saloon-debugging-challenge
|
|
25
|
+
url: https://github.com/brewchetta/ph2-raccoon-saloon-debugging-challenge.git
|
|
26
|
+
default_branch: main
|
|
27
|
+
answer_branch: solution
|
|
28
|
+
pin: 72f52d697b6f40fb41b8f708a76cbc4540accf85
|
|
29
|
+
|
|
30
|
+
- name: react-debug-practice
|
|
31
|
+
url: https://github.com/ArohanD/react-debug-practice.git
|
|
32
|
+
default_branch: master
|
|
33
|
+
answer_branch: solution
|
|
34
|
+
pin: 7400054f7add23c2ec401152197c2056f2b3d7a6
|
|
35
|
+
|
|
36
|
+
# The 3 Tier-1 hard cases, scored individually. Real upstream provenance found (buggy =
|
|
37
|
+
# parent of the fix commit, fixed = the fix commit). Already constructed locally in
|
|
38
|
+
# dawson/ with buggy+fixed branches; URL-listable once hosted or reconstructed from upstream.
|
|
39
|
+
pending:
|
|
40
|
+
- name: cpython-timsort
|
|
41
|
+
upstream: https://github.com/python/cpython.git
|
|
42
|
+
provenance: "listsort merge_collapse run-stack invariant (same class as the Java TimSort bug)"
|
|
43
|
+
default_branch: buggy
|
|
44
|
+
answer_branch: fixed
|
|
45
|
+
status: "host dawson copy, or reconstruct buggy=fix^ / fixed=fix from upstream"
|
|
46
|
+
- name: cpython-asyncio-cond
|
|
47
|
+
upstream: https://github.com/python/cpython.git
|
|
48
|
+
provenance: "Condition.notify vs Task.cancel lost wakeup — python/cpython#112202"
|
|
49
|
+
default_branch: buggy
|
|
50
|
+
answer_branch: fixed
|
|
51
|
+
status: "host dawson copy, or reconstruct from the #112202 fix commit"
|
|
52
|
+
- name: sudo-baron-samedit
|
|
53
|
+
upstream: https://github.com/sudo-project/sudo.git
|
|
54
|
+
provenance: "CVE-2021-3156 heap overflow; fix 1f8638577d0c80a4ff864a2aad80a0d95488e9a8 (+ b301b46b79c6e2a76d530fa36d05992e74952ee8)"
|
|
55
|
+
default_branch: buggy
|
|
56
|
+
answer_branch: fixed
|
|
57
|
+
status: "host dawson copy, or reconstruct buggy=fix^ / fixed=fix from upstream"
|
|
58
|
+
|
|
59
|
+
# NOT part of the eval. Present in dawson/ but never scored. Listed for reference only;
|
|
60
|
+
# fetch.sh ignores this section.
|
|
61
|
+
extras:
|
|
62
|
+
- name: D-Bug
|
|
63
|
+
url: https://github.com/DSC-SIST/D-Bug.git
|
|
64
|
+
default_branch: main
|
|
65
|
+
answer_branch: solution
|
|
66
|
+
- name: matlab-debugging
|
|
67
|
+
url: https://github.com/AaltoSciComp/matlab-debugging.git
|
|
68
|
+
default_branch: main
|
|
69
|
+
answer_branch: solutions
|
|
70
|
+
- name: node-debug-challenge
|
|
71
|
+
url: https://github.com/AlecxMoritz/node-debug-challenge.git
|
|
72
|
+
default_branch: master
|
|
73
|
+
answer_branch: instructor
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Real corpus config: 3 easy repos run through crg-debug on the haiku leg.
|
|
2
|
+
# Fetch first: bash corpora/fetch.sh
|
|
3
|
+
# Then: proofbench run --config corpora/proofbench.yaml
|
|
4
|
+
#
|
|
5
|
+
# The 3 Tier-1 hard cases (cpython-timsort, cpython-asyncio-cond, sudo-baron-samedit)
|
|
6
|
+
# are `pending` in manifest.yaml — add them here once hosted or reconstructed.
|
|
7
|
+
# To self-improve a skill, add `subject:` + `synth:` and set `models` to a weak leg.
|
|
8
|
+
runner: claude
|
|
9
|
+
models: [haiku]
|
|
10
|
+
judge_model: opus
|
|
11
|
+
objective: macro_recall
|
|
12
|
+
concurrency: 3
|
|
13
|
+
max_turns: 200
|
|
14
|
+
|
|
15
|
+
judge: ../prompts/judge.md
|
|
16
|
+
|
|
17
|
+
# BugFix (already validated via the deterministic Workflow in Round 4) and the duplicate
|
|
18
|
+
# sf-wdi-24 debugging-challenges are excluded; these 3 are the active sweep.
|
|
19
|
+
#
|
|
20
|
+
# invoke drives the crg-debugger SUBAGENT (sequential, single in-context session), not the
|
|
21
|
+
# background /crg-debug Workflow: the subagent honors --model haiku and applies fixes
|
|
22
|
+
# synchronously, so capture sees a real git diff to grade. The background Workflow returns a
|
|
23
|
+
# detection report and lands no diff headless, and runs its own internal models.
|
|
24
|
+
corpus:
|
|
25
|
+
- { name: debugging-challenge, path: repos/debugging-challenge, invoke: "Use the crg-debugger subagent (Task tool, subagent_type 'crg-debug:crg-debugger') to find and fix every functional bug in this repository. Apply all fixes directly to the source files. Do not ask for confirmation.", default_branch: master, answer_branch: working }
|
|
26
|
+
- { name: ph2-raccoon-saloon-debugging-challenge, path: repos/ph2-raccoon-saloon-debugging-challenge, invoke: "Use the crg-debugger subagent (Task tool, subagent_type 'crg-debug:crg-debugger') to find and fix every functional bug in this repository. Apply all fixes directly to the source files. Do not ask for confirmation.", default_branch: main, answer_branch: solution }
|
|
27
|
+
- { name: react-debug-practice, path: repos/react-debug-practice, invoke: "Use the crg-debugger subagent (Task tool, subagent_type 'crg-debug:crg-debugger') to find and fix every functional bug in this repository. Apply all fixes directly to the source files. Do not ask for confirmation.", default_branch: master, answer_branch: solution }
|
|
@@ -8,6 +8,14 @@ import subprocess
|
|
|
8
8
|
_EXCLUDES = [":(exclude,glob)**/node_modules/**", ":(exclude,glob)**/.venv/**",
|
|
9
9
|
":(exclude,glob)**/__pycache__/**", ":(exclude,glob)**/.code-review-graph/**"]
|
|
10
10
|
|
|
11
|
+
# Multi-assistant MCP scaffolding that `code-review-graph build` scatters at the repo root to
|
|
12
|
+
# advertise its tools. It is deterministic tool scratch, never an agent fix, so it must not
|
|
13
|
+
# count against precision when grading a graph-driven debugger.
|
|
14
|
+
_EXCLUDES += [f":(exclude,glob){p}" for p in (
|
|
15
|
+
".mcp.json", ".opencode.json", ".cursorrules", ".windsurfrules",
|
|
16
|
+
"AGENTS.md", "CLAUDE.md", "GEMINI.md", "QODER.md",
|
|
17
|
+
".claude/**", ".cursor/**", ".gemini/**", ".kiro/**", ".qoder/**")]
|
|
18
|
+
|
|
11
19
|
|
|
12
20
|
def _git(repo, *args):
|
|
13
21
|
return subprocess.run(["git", "-C", str(repo), *args],
|
|
@@ -26,6 +34,9 @@ def ground_truth(repo, default_branch, answer_branch):
|
|
|
26
34
|
return _git(repo, "diff", f"{default_branch}..{answer_branch}").stdout
|
|
27
35
|
|
|
28
36
|
|
|
29
|
-
def capture(repo):
|
|
37
|
+
def capture(repo, baseline):
|
|
38
|
+
"""Diff against the buggy baseline ref, not the working tree: agents may commit their
|
|
39
|
+
fixes. reset() detaches HEAD, so the baseline ref stays pinned while agent commits move
|
|
40
|
+
HEAD, and this diff captures the fix whether it was committed or left uncommitted."""
|
|
30
41
|
_git(repo, "add", "-N", ".")
|
|
31
|
-
return _git(repo, "diff", "--", ".", *_EXCLUDES).stdout
|
|
42
|
+
return _git(repo, "diff", baseline, "--", ".", *_EXCLUDES).stdout
|
|
@@ -24,19 +24,48 @@ def _ground_truth(cfg, run_dir):
|
|
|
24
24
|
def _run_and_grade(cfg, task, model, gt, out_dir):
|
|
25
25
|
readme, answer = gt[task.name]
|
|
26
26
|
capture.reset(task.path, task.default_branch)
|
|
27
|
-
runner.run(cfg.runner, task.invoke, model, cfg.max_turns, task.path, out_dir / "run.log")
|
|
28
|
-
|
|
27
|
+
meta = runner.run(cfg.runner, task.invoke, model, cfg.max_turns, task.path, out_dir / "run.log")
|
|
28
|
+
cost = f", ${meta['cost_usd']:.3f}" if meta["cost_usd"] is not None else ""
|
|
29
|
+
log(f"[{model}] {task.name}: {meta['seconds']}s{cost}"
|
|
30
|
+
+ (f", {meta['turns']} turns" if meta["turns"] else ""))
|
|
31
|
+
agent_diff = capture.capture(task.path, task.default_branch)
|
|
29
32
|
(out_dir / "agent.diff").write_text(agent_diff)
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
if not agent_diff.strip():
|
|
34
|
+
log(f"\033[33m[{model}] {task.name}: EMPTY diff — agent changed nothing; "
|
|
35
|
+
f"excluded from macro (check the invoke if this is unexpected)\033[0m")
|
|
36
|
+
grade = {"task": task.name, "empty_capture": True}
|
|
37
|
+
else:
|
|
38
|
+
grade = judge.grade(cfg.judge_prompt, readme, answer, agent_diff, cfg.judge_model)
|
|
39
|
+
grade["task"] = task.name
|
|
40
|
+
grade["_meta"] = meta
|
|
32
41
|
(out_dir / "grade.json").write_text(json.dumps(grade, indent=2))
|
|
33
42
|
return grade
|
|
34
43
|
|
|
35
44
|
|
|
45
|
+
def _resources(grades):
|
|
46
|
+
"""Sum cost + wall-clock across the whole leg, empty captures included: a run that
|
|
47
|
+
changed nothing still spent money and time. cost_usd is None when no leg task reported
|
|
48
|
+
it (a non-claude runner)."""
|
|
49
|
+
metas = [g.get("_meta", {}) for g in grades]
|
|
50
|
+
costs = [m["cost_usd"] for m in metas if m.get("cost_usd") is not None]
|
|
51
|
+
return {"seconds": round(sum(m.get("seconds") or 0 for m in metas), 1),
|
|
52
|
+
"cost_usd": round(sum(costs), 4) if costs else None,
|
|
53
|
+
"turns": sum(m.get("turns") or 0 for m in metas) or None}
|
|
54
|
+
|
|
55
|
+
|
|
36
56
|
def aggregate(grades):
|
|
37
|
-
shared
|
|
38
|
-
|
|
39
|
-
|
|
57
|
+
"""Macro-average the shared numeric fields. Empty-capture tasks carry no metrics: they
|
|
58
|
+
are excluded from the average and listed separately so they never become a silent zero."""
|
|
59
|
+
graded = [g for g in grades if not g.get("empty_capture")]
|
|
60
|
+
out = {"tasks": {g["task"]: judge.numeric_fields(g) for g in grades}}
|
|
61
|
+
empty = [g["task"] for g in grades if g.get("empty_capture")]
|
|
62
|
+
if empty:
|
|
63
|
+
out["empty_captures"] = empty
|
|
64
|
+
if graded:
|
|
65
|
+
shared = set.intersection(*(set(judge.numeric_fields(g)) for g in graded))
|
|
66
|
+
out.update({f"macro_{f}": sum(g[f] for g in graded) / len(graded) for f in shared})
|
|
67
|
+
out["resources"] = _resources(grades)
|
|
68
|
+
return out
|
|
40
69
|
|
|
41
70
|
|
|
42
71
|
def bench(cfg, run_dir, models):
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
A built-in name (``claude``) maps to a command builder. Any other value is treated as
|
|
4
4
|
a path to an executable invoked as ``<runner> <invoke> <model>`` — bring your own agent."""
|
|
5
|
+
import json
|
|
5
6
|
import subprocess
|
|
7
|
+
import time
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
def _claude(invoke, model, max_turns):
|
|
@@ -14,12 +16,27 @@ def _claude(invoke, model, max_turns):
|
|
|
14
16
|
BUILTINS = {"claude": _claude}
|
|
15
17
|
|
|
16
18
|
|
|
19
|
+
def _claude_meta(stdout):
|
|
20
|
+
"""Pull cost + turns from a ``claude --output-format json`` envelope. Observability,
|
|
21
|
+
not a graded metric, so this never dies: a custom runner or malformed envelope yields
|
|
22
|
+
None and the wall-clock seconds still stand."""
|
|
23
|
+
try:
|
|
24
|
+
env = json.loads(stdout[stdout.index("{"):stdout.rindex("}") + 1])
|
|
25
|
+
return env.get("total_cost_usd"), env.get("num_turns")
|
|
26
|
+
except (ValueError, json.JSONDecodeError):
|
|
27
|
+
return None, None
|
|
28
|
+
|
|
29
|
+
|
|
17
30
|
def run(runner, invoke, model, max_turns, cwd, log_path):
|
|
18
|
-
"""Execute one headless attempt in ``cwd``; capture all output to ``log_path
|
|
31
|
+
"""Execute one headless attempt in ``cwd``; capture all output to ``log_path`` and
|
|
32
|
+
return its ``{seconds, cost_usd, turns}``.
|
|
19
33
|
|
|
20
34
|
The agent's exit status is advisory — what gets graded is the artifact it leaves
|
|
21
35
|
behind (see capture), never the agent's own claim of success."""
|
|
22
36
|
argv = BUILTINS[runner](invoke, model, max_turns) if runner in BUILTINS \
|
|
23
37
|
else [runner, invoke, model]
|
|
24
|
-
|
|
25
|
-
|
|
38
|
+
start = time.time()
|
|
39
|
+
proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True)
|
|
40
|
+
open(log_path, "w").write(proc.stdout + proc.stderr)
|
|
41
|
+
cost, turns = _claude_meta(proc.stdout) if runner in BUILTINS else (None, None)
|
|
42
|
+
return {"seconds": round(time.time() - start, 1), "cost_usd": cost, "turns": turns}
|
|
@@ -21,6 +21,7 @@ def test_bench_wires_end_to_end(tmp_path, monkeypatch):
|
|
|
21
21
|
def fake_run(runner_name, invoke, model, max_turns, cwd, log_path):
|
|
22
22
|
(Path(cwd) / "lib.py").write_text(FIXED)
|
|
23
23
|
Path(log_path).write_text("stub")
|
|
24
|
+
return {"seconds": 1.0, "cost_usd": 0.01, "turns": 2}
|
|
24
25
|
monkeypatch.setattr(runner, "run", fake_run)
|
|
25
26
|
|
|
26
27
|
# stub judge: credit recall when the agent actually changed something
|
|
@@ -32,5 +33,6 @@ def test_bench_wires_end_to_end(tmp_path, monkeypatch):
|
|
|
32
33
|
leg = board[cfg.models[0]]
|
|
33
34
|
assert leg["macro_recall"] == 1.0
|
|
34
35
|
assert leg["macro_precision"] == 1.0
|
|
36
|
+
assert leg["resources"] == {"seconds": 2.0, "cost_usd": 0.02, "turns": 4}
|
|
35
37
|
saved = json.loads((tmp_path / "scoreboard.json").read_text())
|
|
36
38
|
assert set(saved[cfg.models[0]]["tasks"]) == {"primes", "ranges"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|