proofbench 0.2.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. proofbench-0.3.0/.github/workflows/ci.yml +41 -0
  2. proofbench-0.3.0/.github/workflows/release.yml +28 -0
  3. {proofbench-0.2.1 → proofbench-0.3.0}/.gitignore +1 -0
  4. proofbench-0.3.0/CORPUS.md +142 -0
  5. proofbench-0.3.0/METHODOLOGY.md +129 -0
  6. proofbench-0.3.0/PKG-INFO +190 -0
  7. proofbench-0.3.0/README.md +177 -0
  8. {proofbench-0.2.1 → proofbench-0.3.0}/corpora/build_hard.py +6 -3
  9. {proofbench-0.2.1 → proofbench-0.3.0}/corpora/fetch.sh +5 -5
  10. {proofbench-0.2.1 → proofbench-0.3.0}/corpora/manifest.yaml +8 -3
  11. {proofbench-0.2.1 → proofbench-0.3.0}/corpora/proofbench.yaml +10 -5
  12. proofbench-0.3.0/docs/README.md +29 -0
  13. proofbench-0.3.0/docs/configuration.md +196 -0
  14. {proofbench-0.2.1 → proofbench-0.3.0}/docs/crg-eval-journey.md +17 -2
  15. {proofbench-0.2.1 → proofbench-0.3.0}/docs/crg-eval-results.md +48 -0
  16. proofbench-0.3.0/docs/io-contracts.md +250 -0
  17. proofbench-0.3.0/docs/optimize.md +143 -0
  18. proofbench-0.3.0/docs/question-tasks.md +185 -0
  19. {proofbench-0.2.1 → proofbench-0.3.0}/examples/crg-debug/README.md +6 -4
  20. {proofbench-0.2.1 → proofbench-0.3.0}/examples/proofbench.demo.yaml +4 -3
  21. proofbench-0.3.0/examples/question-demo/arms/oracle-tier1.py +35 -0
  22. proofbench-0.3.0/examples/question-demo/judge.md +19 -0
  23. proofbench-0.3.0/examples/question-demo/proofbench.question.yaml +21 -0
  24. proofbench-0.3.0/examples/question-demo/questions/lineage.jsonl +3 -0
  25. {proofbench-0.2.1 → proofbench-0.3.0}/proofbench/__init__.py +1 -1
  26. proofbench-0.3.0/proofbench/capture.py +72 -0
  27. proofbench-0.3.0/proofbench/cli.py +88 -0
  28. proofbench-0.3.0/proofbench/config.py +198 -0
  29. proofbench-0.3.0/proofbench/corpus.py +87 -0
  30. proofbench-0.3.0/proofbench/engine.py +112 -0
  31. proofbench-0.3.0/proofbench/judge.py +38 -0
  32. proofbench-0.3.0/proofbench/optimize.py +112 -0
  33. proofbench-0.3.0/proofbench/question.py +157 -0
  34. proofbench-0.3.0/proofbench/runner.py +92 -0
  35. proofbench-0.3.0/proofbench/scoreboard.py +225 -0
  36. proofbench-0.3.0/proofbench/util.py +86 -0
  37. {proofbench-0.2.1 → proofbench-0.3.0}/pyproject.toml +11 -1
  38. proofbench-0.3.0/tests/__init__.py +0 -0
  39. proofbench-0.3.0/tests/conftest.py +28 -0
  40. proofbench-0.3.0/tests/stubs.py +10 -0
  41. proofbench-0.3.0/tests/test_abstention_label.py +108 -0
  42. proofbench-0.3.0/tests/test_capture.py +115 -0
  43. proofbench-0.3.0/tests/test_docs.py +51 -0
  44. proofbench-0.3.0/tests/test_engine.py +143 -0
  45. {proofbench-0.2.1 → proofbench-0.3.0}/tests/test_failloud.py +32 -0
  46. proofbench-0.3.0/tests/test_optimize.py +221 -0
  47. proofbench-0.3.0/tests/test_question.py +199 -0
  48. proofbench-0.3.0/tests/test_question_board.py +167 -0
  49. proofbench-0.3.0/tests/test_runner_args.py +152 -0
  50. proofbench-0.3.0/tests/test_stub_marker.py +157 -0
  51. proofbench-0.3.0/tests/test_trials.py +186 -0
  52. proofbench-0.3.0/tests/test_validation.py +254 -0
  53. proofbench-0.2.1/.github/workflows/release.yml +0 -25
  54. proofbench-0.2.1/CORPUS.md +0 -100
  55. proofbench-0.2.1/METHODOLOGY.md +0 -62
  56. proofbench-0.2.1/PKG-INFO +0 -105
  57. proofbench-0.2.1/README.md +0 -92
  58. proofbench-0.2.1/proofbench/capture.py +0 -42
  59. proofbench-0.2.1/proofbench/cli.py +0 -45
  60. proofbench-0.2.1/proofbench/config.py +0 -68
  61. proofbench-0.2.1/proofbench/engine.py +0 -92
  62. proofbench-0.2.1/proofbench/judge.py +0 -20
  63. proofbench-0.2.1/proofbench/optimize.py +0 -58
  64. proofbench-0.2.1/proofbench/runner.py +0 -42
  65. proofbench-0.2.1/proofbench/util.py +0 -49
  66. proofbench-0.2.1/tests/test_engine.py +0 -38
  67. {proofbench-0.2.1 → proofbench-0.3.0}/LICENSE +0 -0
  68. {proofbench-0.2.1 → proofbench-0.3.0}/examples/sample-corpus/build.sh +0 -0
  69. {proofbench-0.2.1 → proofbench-0.3.0/proofbench}/prompts/judge.md +0 -0
  70. {proofbench-0.2.1 → proofbench-0.3.0/proofbench}/prompts/synth.md +0 -0
@@ -0,0 +1,41 @@
1
+ name: ci
2
+ on:
3
+ push:
4
+ branches: [main]
5
+ pull_request:
6
+ workflow_call:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ jobs:
12
+ test:
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ python: ["3.10", "3.11", "3.12", "3.13"]
18
+ steps:
19
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
20
+ - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
21
+ # Without the demo corpus the end-to-end engine test skips itself and says nothing.
22
+ - run: bash examples/sample-corpus/build.sh
23
+ - run: uv run --python ${{ matrix.python }} pytest -q
24
+
25
+ wheel:
26
+ runs-on: ubuntu-latest
27
+ steps:
28
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
29
+ - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
30
+ - run: uv build
31
+ - run: uvx twine check dist/*
32
+ # The tests run against the checkout. This runs what a user installs: the wheel alone,
33
+ # from an empty directory, has to start, carry its prompts and grade the offline demo.
34
+ - name: smoke test the installed wheel
35
+ run: |
36
+ repo="$PWD"
37
+ wheel="$(ls "$repo"/dist/*.whl)"
38
+ cd "$(mktemp -d)"
39
+ uvx --from "$wheel" proofbench --version
40
+ uvx --from "$wheel" python -c "from importlib.resources import files; assert (files('proofbench') / 'prompts' / 'judge.md').is_file()"
41
+ uvx --from "$wheel" proofbench run --config "$repo/examples/question-demo/proofbench.question.yaml"
@@ -0,0 +1,28 @@
1
+ name: release
2
+ on:
3
+ push:
4
+ tags: ["v*"]
5
+
6
+ permissions:
7
+ contents: read
8
+
9
+ jobs:
10
+ ci:
11
+ uses: ./.github/workflows/ci.yml
12
+
13
+ publish:
14
+ needs: ci
15
+ runs-on: ubuntu-latest
16
+ environment: pypi
17
+ permissions:
18
+ contents: read
19
+ id-token: write # Trusted Publishing (OIDC); no PYPI_TOKEN in CI
20
+ steps:
21
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
22
+ - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
23
+ # A tag that disagrees with the package would publish one version under another's name.
24
+ - name: the tag names the version being published
25
+ run: test "v$(uv run python -c 'import proofbench; print(proofbench.__version__)')" = "$GITHUB_REF_NAME"
26
+ - run: uv build
27
+ - run: uvx twine check dist/*
28
+ - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1
@@ -7,4 +7,5 @@ examples/sample-corpus/repos/
7
7
  corpora/repos/
8
8
  .venv/
9
9
  .pytest_cache/
10
+ .coverage
10
11
  uv.lock
@@ -0,0 +1,142 @@
1
+ # Building a repo corpus
2
+
3
+ How to build repo tasks that separate a good agent from a weak one and do not give their
4
+ answers away. For question tasks, see [docs/question-tasks.md](docs/question-tasks.md).
5
+
6
+ [Back to the README](README.md)
7
+
8
+ ## Contents
9
+
10
+ - [What a task is](#what-a-task-is)
11
+ - [How a task is scored](#how-a-task-is-scored)
12
+ - [Two ways a task leaks its answer](#two-ways-a-task-leaks-its-answer)
13
+ - [A checklist for each task](#a-checklist-for-each-task)
14
+ - [Wiring a task into the config](#wiring-a-task-into-the-config)
15
+ - [Tasks from other people's repositories](#tasks-from-other-peoples-repositories)
16
+
17
+ ## What a task is
18
+
19
+ A task is a git repository with two branches.
20
+
21
+ - **The default branch** holds the broken or unfinished state. The agent starts here.
22
+ - **The answer branch** holds the fix. The agent never sees it.
23
+ - **The reference solution** is the diff between the two. The judge grades against it.
24
+
25
+ ```mermaid
26
+ flowchart LR
27
+ subgraph repo["task: one git repository"]
28
+ A["default_branch<br/>(e.g. main / buggy)"]
29
+ B["answer_branch<br/>(e.g. solution / fixed)"]
30
+ end
31
+ A -- "reference solution<br/>git diff default_branch..answer_branch" --> B
32
+ ```
33
+
34
+ Branch names vary from repo to repo: `main`, `master`, `buggy`, `solution`, `working`. The
35
+ config's `default_branch` and `answer_branch` fields say which is which.
36
+
37
+ A good task has two properties:
38
+
39
+ - **It discriminates.** A capable agent solves it and a weak one does not.
40
+ - **It does not leak.** Nothing but the answer branch reveals the fix.
41
+
42
+ ## How a task is scored
43
+
44
+ ```mermaid
45
+ flowchart TD
46
+ clone["fresh clone of default_branch"] --> run["runner starts the agent"]
47
+ run --> cap["capture: git diff of the clone<br/>= the agent's patch"]
48
+ gt["ground truth:<br/>default_branch..answer_branch"] --> judge
49
+ cap --> judge{{"judge"}}
50
+ judge --> out["recall · precision"]
51
+ ```
52
+
53
+ - **The agent works in a throwaway clone.** It holds the default branch alone and has no
54
+ remote, so the answer cannot be fetched. Your corpus repo is only read.
55
+ - **Recall** is how much of the reference solution the agent reproduced.
56
+ - **Precision** is how much of the agent's patch was needed, not collateral damage.
57
+ - **An agent that changes nothing** is graded on its empty diff and counted in the average.
58
+
59
+ One clear defect per task keeps both numbers easy to read.
60
+
61
+ ## Two ways a task leaks its answer
62
+
63
+ A task leaks when the agent or the judge can reach the answer without doing the work.
64
+
65
+ ```mermaid
66
+ flowchart TD
67
+ task["answer leakage"]
68
+ task --> c["Contamination<br/>the model already memorised the fix"]
69
+ task --> l["Label leakage<br/>the repo points at the fix"]
70
+ c --> cc["famous CVEs · textbook off-by-ones<br/>→ measures memory, not method"]
71
+ l --> ll["'intentional bug' READMEs · '# Buggy' comments<br/>→ judge grades the tell, not the contract"]
72
+ ```
73
+
74
+ ### Contamination: avoid famous defects
75
+
76
+ - A capable model recognises Heartbleed, Shellshock or a textbook off-by-one on sight.
77
+ - When it does, you are measuring what it remembers, not how it debugs.
78
+ - Prefer subtle, real defects whose fix turns on an invariant, a race or a contract.
79
+ - Prefer fixes newer than the model's training data.
80
+
81
+ ### Label leakage: do not point at the answer
82
+
83
+ The judge reads the repo's README. If the README says "this repo has an intentional bug", or
84
+ the broken line carries a `# Buggy` comment, both the agent and the judge read the hint
85
+ instead of the code. In one real run this turned real contract violations into "intended
86
+ design" and quietly corrupted the grades.
87
+
88
+ - Write the README as a real library would. State what each function is supposed to do, never
89
+ that it is wrong.
90
+ - Remove comments that point at the defect. Leave the broken line where it is.
91
+ - Keep the fix in exactly one place: the answer branch.
92
+
93
+ ## A checklist for each task
94
+
95
+ - [ ] The repository root is the task. A folder inside another repository is refused.
96
+ - [ ] The default branch is a branch or tag, and it holds the broken state.
97
+ - [ ] The answer branch differs from the default branch.
98
+ - [ ] Every file the answer branch changes is part of the fix, or is listed under `ignore`.
99
+ - [ ] The README states the contract and does not mention the bug.
100
+ - [ ] No comment points at the defect.
101
+ - [ ] The defect is not famous.
102
+ - [ ] The task has one defect, or one coherent set.
103
+
104
+ `examples/sample-corpus/build.sh` is a small worked example: two repos, each with a `buggy`
105
+ branch, a `fixed` branch, and a README that states the contract.
106
+
107
+ ## Wiring a task into the config
108
+
109
+ ```yaml
110
+ corpus:
111
+ - name: my-task # unique
112
+ path: repos/my-task # relative to the config file
113
+ invoke: "Find and fix the bug in this repository."
114
+ default_branch: buggy
115
+ answer_branch: fixed
116
+ holdout: true # optional; scored by optimize, never shown to the rewrite
117
+ ignore: ["dist/**"] # optional; paths that are not part of the fix
118
+ ```
119
+
120
+ - Every field is described in [docs/configuration.md](docs/configuration.md#a-repo-task).
121
+ - `holdout` is explained in [docs/optimize.md](docs/optimize.md#the-held-out-split).
122
+
123
+ **Check what your answer branch really changes.** Run `git diff --stat default..answer` on
124
+ each task. Answer branches often carry more than the fix:
125
+
126
+ - a rebuilt bundle or other generated output
127
+ - a rewritten README
128
+ - a file of solution notes
129
+
130
+ The judge counts every change in the answer as something the agent should have done. On one
131
+ real task this scored an agent that fixed both bugs at 0.40 recall, because three of the five
132
+ "items" were files like these. List them under `ignore` and they are left out of both sides.
133
+
134
+ ## Tasks from other people's repositories
135
+
136
+ - Do not commit large or licensed corpora into your own repo.
137
+ - List them by URL and clone them at setup.
138
+ - **Pin both branches to exact commits.** If upstream moves its answer branch, an unpinned
139
+ corpus quietly changes what you grade against.
140
+
141
+ `corpora/manifest.yaml` and `corpora/fetch.sh` show the pattern: each entry has a `pin` for
142
+ the default branch and an `answer_pin` for the answer branch.
@@ -0,0 +1,129 @@
1
+ # Methodology
2
+
3
+ Why proofbench is built the way it is. It came out of a long evaluation of one debugging
4
+ skill, and each lesson from that evaluation is now something the tool enforces, not advice
5
+ you have to remember.
6
+
7
+ - The story of that evaluation: [docs/crg-eval-journey.md](docs/crg-eval-journey.md)
8
+ - Its numbers: [docs/crg-eval-results.md](docs/crg-eval-results.md)
9
+
10
+ [Back to the README](README.md)
11
+
12
+ ## Contents
13
+
14
+ - [1. Fail loud or do not measure](#1-fail-loud-or-do-not-measure)
15
+ - [2. The weak model is the signal](#2-the-weak-model-is-the-signal)
16
+ - [3. A rule in a prompt is a request](#3-a-rule-in-a-prompt-is-a-request)
17
+ - [4. The corpus is part of the method](#4-the-corpus-is-part-of-the-method)
18
+ - [5. The thing being measured must not see the answer](#5-the-thing-being-measured-must-not-see-the-answer)
19
+ - [The arc in one line](#the-arc-in-one-line)
20
+
21
+ ## 1. Fail loud or do not measure
22
+
23
+ **What happened.** The most dangerous output of a test harness is a confident wrong number.
24
+ In the original run:
25
+
26
+ - A judge call silently exceeded the operating system's argument limit.
27
+ - Its error was swallowed, and an empty grade flowed onward.
28
+ - The scoreboard reported a clean average of four cells and presented it as five.
29
+
30
+ It was not a crash. It was a plausible lie.
31
+
32
+ **What proofbench does.**
33
+
34
+ - Every empty or non-numeric grade stops the run.
35
+ - Large prompts go to the judge on stdin, never as an argument, so a long diff cannot abort
36
+ the call.
37
+ - Dependency and cache folders are left out of the captured diff. They are never the agent's
38
+ work and can make a prompt enormous.
39
+ - Every git command and every child process is checked. A failure stops the run in one line.
40
+ - The config and the question file are checked whole before anything is paid for.
41
+
42
+ **The same rule in question tasks.** Each of these was a decision worth writing down:
43
+
44
+ - **A missing token count is fatal.** Without `tokens_in` and `tokens_out` there is no tokens
45
+ per correct answer, and a zero there would read as a free answer.
46
+ - **Nothing correct means no price, not a price of zero.** `tokens_per_correct` is undefined
47
+ when nothing was correct. It is reported as null, and reading it as an objective stops the
48
+ run. A zero would rank the worst arm as the cheapest.
49
+ - **The judge reads words, never ids.** An exact match on ids settles a question with no model
50
+ involved. The judge is only reached for prose. Its verdict must be 1 or 0, because partial
51
+ credit would quietly inflate accuracy.
52
+ - **A corpus with nothing answerable gets no key named accuracy.** Calling that number
53
+ accuracy makes an arm that refused everything look perfect. Such a leg reports
54
+ `abstention_rate` instead.
55
+ - **Measured is a positive claim.** A leg counts as a measurement only when the arm says it
56
+ was not canned and says what it cost. Silence is not evidence.
57
+ - **The grid's axes come from the design, never from the data.** A table drawn over the cells
58
+ that happen to be filled looks complete with two cells of fifteen. Every cell prints, and an
59
+ empty one prints as empty.
60
+
61
+ ## 2. The weak model is the signal
62
+
63
+ **What happened.** A loop that improves a skill learns from misses. Frontier models on easy
64
+ tasks miss almost nothing, so they teach nothing.
65
+
66
+ **The consequence.** The weak model is not the liability in this setup. It is the only source
67
+ of signal. Its misses are the curriculum.
68
+
69
+ **What proofbench does.**
70
+
71
+ - `models` is a list, so every run can sweep several models.
72
+ - `optimize` runs against the last entry, the weakest, on purpose.
73
+ - `judge_model` and `synth_model` are held constant. They default to the first entry, so list
74
+ the strongest model first or name them yourself.
75
+ - The model under test never grades its own work or rewrites its own instructions.
76
+
77
+ ## 3. A rule in a prompt is a request
78
+
79
+ **What happened.** The longest stretch of the original work went into fixing a weak model's
80
+ mistakes by writing better instructions. It failed, repeatedly and expensively, in the same
81
+ way every time.
82
+
83
+ - How well a model follows a prompt grows with how strong the model is.
84
+ - So the model that most needs a rule is the one most likely to treat it as optional.
85
+ - You cannot word a weak model into being rigorous.
86
+
87
+ **What proofbench does.** When a rule must hold, it lives in code the model cannot route
88
+ around.
89
+
90
+ - The keep or revert decision in `optimize` is the harness comparing two numbers.
91
+ - "The score held" is not something the model can say. Only the comparison can return it.
92
+
93
+ ## 4. The corpus is part of the method
94
+
95
+ Two findings about the data, not the harness. Both are expanded in [CORPUS.md](CORPUS.md).
96
+
97
+ - **Do not use famous bugs.** A frontier model recognises Heartbleed on sight, and then you
98
+ are measuring memory, not method. Pick subtle, real defects where the reasoning is about an
99
+ invariant or a contract.
100
+ - **Do not let the corpus point at its answers.** In one run the task repos announced their
101
+ own planted bugs in the README and in `# Buggy` comments. The judge read the hints and
102
+ classified real contract violations as intended design. State the true contract and never
103
+ label the defect.
104
+
105
+ ## 5. The thing being measured must not see the answer
106
+
107
+ **What happened.** A later audit found the harness itself leaking.
108
+
109
+ - A question arm was handed the whole question record, answer key included.
110
+ - A repo agent worked in the repository that held the answer branch, one `git checkout` away.
111
+ - The rewrite in `optimize` was scored on the same items whose answers it had been shown.
112
+
113
+ None of these needed a dishonest model. An arm that simply passed its input along, or an
114
+ agent that listed its branches, would have been graded against a key it had been given.
115
+
116
+ **What proofbench does.**
117
+
118
+ - An arm is handed a question's id and text, and nothing else.
119
+ - A repo agent works in a throwaway clone that holds the broken branch alone, with no remote.
120
+ - `optimize` learns from one part of the corpus and is scored on another, marked
121
+ `holdout: true`.
122
+ - The judge runs with no tools and none of your personal configuration, so a diff cannot
123
+ instruct it and a grade does not depend on where you ran the command.
124
+
125
+ ## The arc in one line
126
+
127
+ The evaluation kept writing its own conclusions, round after round. The work was learning to
128
+ stop saying them in a prompt and start building them in code. proofbench is what that looks
129
+ like when it is finished: the lessons are the implementation.
@@ -0,0 +1,190 @@
1
+ Metadata-Version: 2.5
2
+ Name: proofbench
3
+ Version: 0.3.0
4
+ Summary: Config-driven eval harness that grades and self-improves headless agent skills against ground-truth corpora
5
+ Project-URL: Homepage, https://github.com/CodeBlackwell/proofbench
6
+ Author: LeChristopher Blackwell
7
+ License: MIT
8
+ License-File: LICENSE
9
+ Keywords: agent,benchmark,claude,eval,llm,prompt-optimization
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: pyyaml>=6.0
12
+ Description-Content-Type: text/markdown
13
+
14
+ # proofbench
15
+
16
+ proofbench measures whether an agent skill actually works, and can improve the skill without
17
+ letting it grade itself.
18
+
19
+ - You give it tasks you already know the answers to.
20
+ - It runs an agent over them and grades the result against the known answer.
21
+ - With `optimize`, it rewrites the skill from its misses and keeps the rewrite only if the
22
+ score holds.
23
+
24
+ It began as the harness behind [crg-debug](examples/crg-debug/README.md), a debugging skill.
25
+ It works for any skill whose output can be compared to a known answer.
26
+
27
+ ## Contents
28
+
29
+ - [Why it exists](#why-it-exists)
30
+ - [Install](#install)
31
+ - [Quickstart](#quickstart)
32
+ - [How it works](#how-it-works)
33
+ - [A first config](#a-first-config)
34
+ - [Two task kinds](#two-task-kinds)
35
+ - [Commands](#commands)
36
+ - [What a run writes](#what-a-run-writes)
37
+ - [Safety](#safety)
38
+ - [Documentation](#documentation)
39
+ - [License](#license)
40
+
41
+ ## Why it exists
42
+
43
+ Anyone can write a prompt that sounds like a method. The honest test is to measure it against
44
+ known answers, on a model weak enough to fail. Two rules shape the whole tool:
45
+
46
+ 1. **Fail loud or do not measure.** A grade that is empty, not a number, or produced by a
47
+ failed step stops the run. A harness that keeps going reports a confident wrong number.
48
+ 2. **The weak model is the signal.** A loop that learns from misses learns nothing from a
49
+ model that never misses. `optimize` runs against the weakest model on purpose.
50
+
51
+ [METHODOLOGY.md](METHODOLOGY.md) explains where both rules came from.
52
+
53
+ ## Install
54
+
55
+ ```bash
56
+ uv tool install proofbench # or: uvx proofbench
57
+ ```
58
+
59
+ You also need:
60
+
61
+ - **The `claude` CLI on your PATH**, for the default runner and for the judge. The judge uses
62
+ its `--tools` and `--safe-mode` flags, so an old version stops the run with the CLI's error.
63
+ - **git**, for repo tasks.
64
+ - **Python 3.10 or newer.**
65
+
66
+ The install contains the engine and two prompts for repo tasks. The examples and corpus
67
+ scripts live in this repository.
68
+
69
+ ## Quickstart
70
+
71
+ The offline demo needs no API key and no `claude` CLI:
72
+
73
+ ```bash
74
+ git clone https://github.com/CodeBlackwell/proofbench && cd proofbench
75
+ uv run proofbench run --config examples/question-demo/proofbench.question.yaml
76
+ ```
77
+
78
+ The repo demo runs a real agent on two toy repositories and costs a few cents:
79
+
80
+ ```bash
81
+ bash examples/sample-corpus/build.sh # builds two toy repos, each with a bug and a fix
82
+ uv run proofbench run --demo # runs the agent, then grades it against the fix
83
+ ```
84
+
85
+ ## How it works
86
+
87
+ ```mermaid
88
+ flowchart LR
89
+ config["config.yaml"] --> bench["proofbench run"]
90
+ bench --> agent["agent or arm<br/>does the task"]
91
+ agent --> capture["capture<br/>diff or answer"]
92
+ capture --> judge["judge<br/>compares to the answer"]
93
+ judge --> board["scoreboard.json"]
94
+ ```
95
+
96
+ 1. **Read the config.** It is checked whole before anything is paid for.
97
+ 2. **Run the agent** on each task, once per model in `models`.
98
+ 3. **Capture** what the agent produced: a git diff for a repo task, an answer for a question.
99
+ 4. **Grade** it against the known answer.
100
+ 5. **Write the scoreboard**, one entry per model.
101
+
102
+ `optimize` adds a loop on top: baseline, rewrite the skill, re-run, then keep or revert.
103
+ See [docs/optimize.md](docs/optimize.md).
104
+
105
+ ## A first config
106
+
107
+ ```yaml
108
+ runner: claude # the built-in runner; any executable also works
109
+ models: [opus, sonnet, haiku] # strongest first
110
+ judge_model: opus # the same judge for every model
111
+ judge: default # the judge prompt proofbench ships for repo tasks
112
+ corpus:
113
+ - name: primes
114
+ path: examples/sample-corpus/repos/primes
115
+ invoke: "Find and fix the bug in this repository."
116
+ default_branch: buggy # the broken state the agent starts from
117
+ answer_branch: fixed # the known answer
118
+ ```
119
+
120
+ - Paths are relative to the config file.
121
+ - Every key, its default and the checks made at load are in
122
+ [docs/configuration.md](docs/configuration.md).
123
+
124
+ ## Two task kinds
125
+
126
+ One corpus holds one kind.
127
+
128
+ | | `repo` (the default) | `question` |
129
+ |---|---|---|
130
+ | The task | Fix a repository | Answer a question |
131
+ | The known answer | A branch holding the fix | An answer key in a JSONL file |
132
+ | What runs | `claude`, or your executable, inside a throwaway clone | Your executable, called an arm |
133
+ | What is graded | The diff the agent leaves | The answer the arm prints |
134
+ | Who grades | The judge model | An exact id match first, the judge only for prose |
135
+ | Main numbers | `macro_recall`, `macro_precision` | `accuracy`, `pass^k`, `tokens_per_correct` |
136
+ | Read more | [CORPUS.md](CORPUS.md) | [docs/question-tasks.md](docs/question-tasks.md) |
137
+
138
+ ## Commands
139
+
140
+ | Command | What it does |
141
+ |---|---|
142
+ | `proofbench run --config FILE` | Run every model over the corpus and write a scoreboard |
143
+ | `proofbench optimize --config FILE` | Rewrite the `subject` from its misses; keep the rewrite only if the score holds |
144
+ | `proofbench run --demo` | Run the repo demo (needs a clone of this repository) |
145
+ | `proofbench --version` | Print the version |
146
+
147
+ - `--run-id NAME` names the run folder. Without it the folder is named for the start time.
148
+ - The exit code is 1 on any hard stop, and the reason is printed in one line.
149
+
150
+ ## What a run writes
151
+
152
+ Everything goes under `runs/<run-id>/`, in the directory you ran the command from:
153
+
154
+ - `scoreboard.json`: the numbers, one entry per model.
155
+ - One folder per model and task, holding the agent's log, what it produced, and its grade.
156
+
157
+ A folder that already holds a run is refused, so two runs never mix. The full layout and the
158
+ shape of each file are in [docs/io-contracts.md](docs/io-contracts.md).
159
+
160
+ ## Safety
161
+
162
+ - **The agent runs with permissions skipped.** A repo task starts `claude` with
163
+ `--dangerously-skip-permissions`. It works in a throwaway clone, but the clone is not a
164
+ sandbox: the agent can still reach the rest of your machine. Run untrusted corpora in a
165
+ container or VM.
166
+ - **Your corpus repos are only read.** proofbench never resets, cleans or commits in them.
167
+ - **The agent cannot reach the answer through git.** The clone holds the broken branch alone
168
+ and has no remote.
169
+ - **The judge has no tools.** It reads text the agent wrote, so it runs with no tools and none
170
+ of your `CLAUDE.md`, hooks, plugins or MCP servers.
171
+ - **`optimize` edits your subject file.** The original is put back unless the rewrite is kept,
172
+ including when the run crashes or is interrupted.
173
+
174
+ ## Documentation
175
+
176
+ | Page | Read it when you want to |
177
+ |---|---|
178
+ | [docs/configuration.md](docs/configuration.md) | Look up a config key, a default, or a load-time error |
179
+ | [docs/io-contracts.md](docs/io-contracts.md) | Write a runner or an arm, or read the files a run writes |
180
+ | [docs/question-tasks.md](docs/question-tasks.md) | Build a question corpus and understand its numbers |
181
+ | [docs/optimize.md](docs/optimize.md) | Use the self-improving loop and its held-out split |
182
+ | [CORPUS.md](CORPUS.md) | Build repo tasks that are hard to game and do not leak their answer |
183
+ | [METHODOLOGY.md](METHODOLOGY.md) | Understand why the tool is built the way it is |
184
+ | [examples/crg-debug/README.md](examples/crg-debug/README.md) | See a real config for a real skill |
185
+ | [docs/crg-eval-journey.md](docs/crg-eval-journey.md) | Read the story of the evaluation that produced the tool |
186
+ | [docs/crg-eval-results.md](docs/crg-eval-results.md) | See that evaluation's tables and per-round verdicts |
187
+
188
+ ## License
189
+
190
+ MIT.