precheck 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- precheck-0.1.0/LICENSE +21 -0
- precheck-0.1.0/PKG-INFO +227 -0
- precheck-0.1.0/README.md +207 -0
- precheck-0.1.0/precheck/__init__.py +5 -0
- precheck-0.1.0/precheck/__main__.py +6 -0
- precheck-0.1.0/precheck/cli.py +210 -0
- precheck-0.1.0/precheck/core.py +173 -0
- precheck-0.1.0/precheck/runtime.py +19 -0
- precheck-0.1.0/precheck/vacuity.py +210 -0
- precheck-0.1.0/precheck.egg-info/PKG-INFO +227 -0
- precheck-0.1.0/precheck.egg-info/SOURCES.txt +16 -0
- precheck-0.1.0/precheck.egg-info/dependency_links.txt +1 -0
- precheck-0.1.0/precheck.egg-info/entry_points.txt +2 -0
- precheck-0.1.0/precheck.egg-info/top_level.txt +1 -0
- precheck-0.1.0/pyproject.toml +42 -0
- precheck-0.1.0/setup.cfg +4 -0
- precheck-0.1.0/tests/test_core.py +121 -0
- precheck-0.1.0/tests/test_vacuity.py +139 -0
precheck-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Simin Yuan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
precheck-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: precheck
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Make an agent prove its claims with checks it was not allowed to write. The check is frozen before the run, and audited for the ability to fail at all.
|
|
5
|
+
Author: Simin Yuan
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/simin-yuan/precheck
|
|
8
|
+
Project-URL: Issues, https://github.com/simin-yuan/precheck/issues
|
|
9
|
+
Keywords: agents,ai-agents,verification,audit,mutation-testing,llm,ci,trust,evidence
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Classifier: Topic :: Software Development :: Testing
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
<p align="center">
|
|
22
|
+
<img src="assets/banner.svg" alt="precheck" width="880">
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
<h1 align="center">precheck</h1>
|
|
26
|
+
|
|
27
|
+
<p align="center"><b>Make an agent prove its claims with checks it was not allowed to write.</b></p>
|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
<a href="https://github.com/simin-yuan/precheck/actions/workflows/tests.yml"><img alt="tests" src="https://github.com/simin-yuan/precheck/actions/workflows/tests.yml/badge.svg"></a>
|
|
31
|
+
<img alt="license" src="https://img.shields.io/badge/license-MIT-blue">
|
|
32
|
+
<img alt="python" src="https://img.shields.io/badge/python-3.8%2B-blue">
|
|
33
|
+
<img alt="dependencies" src="https://img.shields.io/badge/dependencies-0-brightgreen">
|
|
34
|
+
<img alt="network" src="https://img.shields.io/badge/network-none-lightgrey">
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
An agent finishes and reports success. Today you can believe it, or re-read
|
|
40
|
+
every diff yourself. `precheck` is a third option.
|
|
41
|
+
|
|
42
|
+
It does two things a test runner does not:
|
|
43
|
+
|
|
44
|
+
1. **It freezes the check before the run.** The acceptance check is written and
|
|
45
|
+
hash-locked *before* the work happens, so the actor cannot tailor the test to
|
|
46
|
+
whatever it ended up producing. Edit the check afterwards and every verdict
|
|
47
|
+
after that point is void — provably, not by convention.
|
|
48
|
+
2. **It asks whether the check could have failed at all.** After a check passes,
|
|
49
|
+
`precheck` mutates the artifact it was judging and runs it again. A check that
|
|
50
|
+
still passes on broken input was never evidence for anything.
|
|
51
|
+
|
|
52
|
+
## What that looks like
|
|
53
|
+
|
|
54
|
+
Every line below is real output from `python demo.py` in this repository. (The
|
|
55
|
+
trailing `(exit N)` annotations it prints, and its temp-directory line, are
|
|
56
|
+
omitted here.)
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
$ precheck register
|
|
60
|
+
froze 1 commitment(s) at seq=1 sha256=e2d4561aa9f0
|
|
61
|
+
|
|
62
|
+
$ precheck settle
|
|
63
|
+
PASS the deployment config is valid and safe to ship
|
|
64
|
+
|
|
65
|
+
1/1 passed (seq=2)
|
|
66
|
+
|
|
67
|
+
$ precheck audit
|
|
68
|
+
ran 4 mutation(s) across the declared artefacts
|
|
69
|
+
|
|
70
|
+
2 check/mutation pair(s) survived -- these prove nothing yet:
|
|
71
|
+
? C1 [blank-value on config.json]
|
|
72
|
+
replicas: 3 -> 0
|
|
73
|
+
the check still exited 0
|
|
74
|
+
? C1 [flip-number on config.json]
|
|
75
|
+
"replicas": 3, -> "replicas": 10,
|
|
76
|
+
the check still exited 0
|
|
77
|
+
|
|
78
|
+
This is a question list, not a bug list. Some survivors are legitimate.
|
|
79
|
+
|
|
80
|
+
$ precheck verify
|
|
81
|
+
. commitments unchanged since seq=1 (e2d4561aa9f0)
|
|
82
|
+
|
|
83
|
+
3 entries; chain consistent
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The check caught a broken file. It did not catch a config that would take
|
|
87
|
+
production down: `replicas: 3` became `0` and the check still said yes. That is
|
|
88
|
+
the failure mode this tool exists for.
|
|
89
|
+
|
|
90
|
+
## Install
|
|
91
|
+
|
|
92
|
+
Not on PyPI yet. Two ways to use it today, both of which work right now:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
# install straight from the repository
|
|
96
|
+
pip install git+https://github.com/simin-yuan/precheck.git
|
|
97
|
+
|
|
98
|
+
# or run it in place -- it is pure standard library, no install needed
|
|
99
|
+
git clone https://github.com/simin-yuan/precheck
|
|
100
|
+
cd precheck && python demo.py
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Zero runtime dependencies, standard library only, no network calls.
|
|
104
|
+
Python 3.8+.
|
|
105
|
+
|
|
106
|
+
## Use it
|
|
107
|
+
|
|
108
|
+
Three commands you run in order, and one anyone can run afterwards.
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
precheck init # writes .precheck/commitments.json
|
|
112
|
+
# ... edit it: state the claim, the check, and the artifacts it is about
|
|
113
|
+
precheck register # freeze it -- do this BEFORE the work runs
|
|
114
|
+
# ... whatever produces the artifact runs here ...
|
|
115
|
+
precheck settle # run the frozen checks, record PASS / FAIL / TIMEOUT
|
|
116
|
+
precheck audit # mutate the artifacts; find checks that cannot fail
|
|
117
|
+
precheck verify # walk the hash chain; detect edited history
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`settle` exits `2` on any failure. `audit --strict` exits `3` if a check survived
|
|
121
|
+
a mutation. `verify` exits `1` if the chain is broken. All three drop straight
|
|
122
|
+
into CI.
|
|
123
|
+
|
|
124
|
+
### As a GitHub Action
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
jobs:
|
|
128
|
+
verify-the-agent:
|
|
129
|
+
runs-on: ubuntu-latest
|
|
130
|
+
steps:
|
|
131
|
+
- uses: actions/checkout@v4
|
|
132
|
+
- uses: simin-yuan/precheck@main
|
|
133
|
+
with:
|
|
134
|
+
command: audit # settle, then mutate and re-run
|
|
135
|
+
strict: "true" # fail the job if a check could not fail
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Put `precheck register` in the job *before* the step that produces the artifact.
|
|
139
|
+
A check frozen after the fact is reported as `NOT_REGISTERED`, and `verify` fails.
|
|
140
|
+
|
|
141
|
+
### The commitments file
|
|
142
|
+
|
|
143
|
+
```json
|
|
144
|
+
{
|
|
145
|
+
"version": 1,
|
|
146
|
+
"commitments": [
|
|
147
|
+
{
|
|
148
|
+
"id": "C1",
|
|
149
|
+
"statement": "the deployment config is valid and safe to ship",
|
|
150
|
+
"check": "python check_config.py",
|
|
151
|
+
"artifacts": ["config.json"]
|
|
152
|
+
}
|
|
153
|
+
]
|
|
154
|
+
}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
`check` is any shell command whose exit code decides the claim. `artifacts` are
|
|
158
|
+
the files that claim is about — the audit mutates those.
|
|
159
|
+
|
|
160
|
+
### Mutations
|
|
161
|
+
|
|
162
|
+
`drop-line` · `blank-value` · `flip-number` · `truncate`
|
|
163
|
+
|
|
164
|
+
Deterministic given `--seed`, so a survivor is reproducible and can be argued
|
|
165
|
+
about. Binary files are never mutated.
|
|
166
|
+
|
|
167
|
+
## Why not just use pytest?
|
|
168
|
+
|
|
169
|
+
`pytest` asks *is the code right?* `precheck` asks *is your proof right?* They
|
|
170
|
+
are different questions, and the second one currently has no tooling:
|
|
171
|
+
|
|
172
|
+
- `pytest` has no opinion on who wrote the test, or when. `precheck` refuses a
|
|
173
|
+
check that was written after the result it judges.
|
|
174
|
+
- `pytest` cannot tell you that a passing test would also pass on a broken
|
|
175
|
+
file. `precheck` mutates and re-runs to find out.
|
|
176
|
+
- `pytest` results live in your terminal. `precheck` records them in a chain a
|
|
177
|
+
third party can verify without trusting you.
|
|
178
|
+
|
|
179
|
+
You keep using pytest. You point `precheck` at it.
|
|
180
|
+
|
|
181
|
+
## How the tamper-evidence works
|
|
182
|
+
|
|
183
|
+
Each ledger entry is canonical JSON (sorted keys, fixed separators), and holds
|
|
184
|
+
`prev` — the previous entry's sha256 — plus its own hash. Editing any byte of any
|
|
185
|
+
entry re-hashes to something different, and deleting an entry breaks the link
|
|
186
|
+
after it. `verify` re-walks the whole file and reports:
|
|
187
|
+
|
|
188
|
+
| finding | meaning |
|
|
189
|
+
|---|---|
|
|
190
|
+
| `HASH_MISMATCH` | an entry's body was edited |
|
|
191
|
+
| `BROKEN_LINK` | an entry was deleted, or its predecessor was replaced |
|
|
192
|
+
| `SEQ_GAP` | entries were removed from the middle |
|
|
193
|
+
| `COMMITMENTS_MODIFIED` | the check was rewritten after it was frozen |
|
|
194
|
+
| `NOT_REGISTERED` | checks were never frozen at all |
|
|
195
|
+
|
|
196
|
+
Commit `.precheck/` with your code. That is the point — a reader can re-run
|
|
197
|
+
`precheck verify` on your repository and see for themselves.
|
|
198
|
+
|
|
199
|
+
## Limits — read this before you trust it
|
|
200
|
+
|
|
201
|
+
- **A surviving mutation is a question, not a bug.** The mutated line may be
|
|
202
|
+
genuinely irrelevant to the claim. `precheck` will not tell you which; a human
|
|
203
|
+
decides.
|
|
204
|
+
- **Surviving every mutation is not proof that a check is vacuous.** The
|
|
205
|
+
mutations are a fixed sample of four. A check can be useless in ways this
|
|
206
|
+
sample never touches.
|
|
207
|
+
- **The chain proves history was not edited after the fact. It does not prove
|
|
208
|
+
the first entry was honest.** Whoever writes the first `register` entry can
|
|
209
|
+
still write a weak check. Pre-registration raises the cost of gaming the
|
|
210
|
+
result; it does not make gaming impossible.
|
|
211
|
+
- **A weak artifact list defeats it.** If you declare no artifacts, there is
|
|
212
|
+
nothing to mutate and `audit` will honestly report that it examined nothing.
|
|
213
|
+
Declare the files the claim is actually about.
|
|
214
|
+
- **Exit codes only.** `precheck` has no idea what your check *means*. A check
|
|
215
|
+
that prints a lie and exits 0 is a check that passes.
|
|
216
|
+
|
|
217
|
+
## Status
|
|
218
|
+
|
|
219
|
+
Alpha. 19 unit tests over the chain, the freeze, and the audit; a runnable
|
|
220
|
+
end-to-end demo; no dependencies. The audit's mutation set is deliberately small
|
|
221
|
+
and readable — adding mutations that you cannot explain to a reader would make
|
|
222
|
+
the output less trustworthy, not more.
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
MIT © 2026 Simin Yuan
|
|
227
|
+
|
precheck-0.1.0/README.md
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/banner.svg" alt="precheck" width="880">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">precheck</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center"><b>Make an agent prove its claims with checks it was not allowed to write.</b></p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://github.com/simin-yuan/precheck/actions/workflows/tests.yml"><img alt="tests" src="https://github.com/simin-yuan/precheck/actions/workflows/tests.yml/badge.svg"></a>
|
|
11
|
+
<img alt="license" src="https://img.shields.io/badge/license-MIT-blue">
|
|
12
|
+
<img alt="python" src="https://img.shields.io/badge/python-3.8%2B-blue">
|
|
13
|
+
<img alt="dependencies" src="https://img.shields.io/badge/dependencies-0-brightgreen">
|
|
14
|
+
<img alt="network" src="https://img.shields.io/badge/network-none-lightgrey">
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
An agent finishes and reports success. Today you can believe it, or re-read
|
|
20
|
+
every diff yourself. `precheck` is a third option.
|
|
21
|
+
|
|
22
|
+
It does two things a test runner does not:
|
|
23
|
+
|
|
24
|
+
1. **It freezes the check before the run.** The acceptance check is written and
|
|
25
|
+
hash-locked *before* the work happens, so the actor cannot tailor the test to
|
|
26
|
+
whatever it ended up producing. Edit the check afterwards and every verdict
|
|
27
|
+
after that point is void — provably, not by convention.
|
|
28
|
+
2. **It asks whether the check could have failed at all.** After a check passes,
|
|
29
|
+
`precheck` mutates the artifact it was judging and runs it again. A check that
|
|
30
|
+
still passes on broken input was never evidence for anything.
|
|
31
|
+
|
|
32
|
+
## What that looks like
|
|
33
|
+
|
|
34
|
+
Every line below is real output from `python demo.py` in this repository. (The
|
|
35
|
+
trailing `(exit N)` annotations it prints, and its temp-directory line, are
|
|
36
|
+
omitted here.)
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
$ precheck register
|
|
40
|
+
froze 1 commitment(s) at seq=1 sha256=e2d4561aa9f0
|
|
41
|
+
|
|
42
|
+
$ precheck settle
|
|
43
|
+
PASS the deployment config is valid and safe to ship
|
|
44
|
+
|
|
45
|
+
1/1 passed (seq=2)
|
|
46
|
+
|
|
47
|
+
$ precheck audit
|
|
48
|
+
ran 4 mutation(s) across the declared artefacts
|
|
49
|
+
|
|
50
|
+
2 check/mutation pair(s) survived -- these prove nothing yet:
|
|
51
|
+
? C1 [blank-value on config.json]
|
|
52
|
+
replicas: 3 -> 0
|
|
53
|
+
the check still exited 0
|
|
54
|
+
? C1 [flip-number on config.json]
|
|
55
|
+
"replicas": 3, -> "replicas": 10,
|
|
56
|
+
the check still exited 0
|
|
57
|
+
|
|
58
|
+
This is a question list, not a bug list. Some survivors are legitimate.
|
|
59
|
+
|
|
60
|
+
$ precheck verify
|
|
61
|
+
. commitments unchanged since seq=1 (e2d4561aa9f0)
|
|
62
|
+
|
|
63
|
+
3 entries; chain consistent
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The check caught a broken file. It did not catch a config that would take
|
|
67
|
+
production down: `replicas: 3` became `0` and the check still said yes. That is
|
|
68
|
+
the failure mode this tool exists for.
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
Not on PyPI yet. Two ways to use it today, both of which work right now:
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
# install straight from the repository
|
|
76
|
+
pip install git+https://github.com/simin-yuan/precheck.git
|
|
77
|
+
|
|
78
|
+
# or run it in place -- it is pure standard library, no install needed
|
|
79
|
+
git clone https://github.com/simin-yuan/precheck
|
|
80
|
+
cd precheck && python demo.py
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Zero runtime dependencies, standard library only, no network calls.
|
|
84
|
+
Python 3.8+.
|
|
85
|
+
|
|
86
|
+
## Use it
|
|
87
|
+
|
|
88
|
+
Three commands you run in order, and one anyone can run afterwards.
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
precheck init # writes .precheck/commitments.json
|
|
92
|
+
# ... edit it: state the claim, the check, and the artifacts it is about
|
|
93
|
+
precheck register # freeze it -- do this BEFORE the work runs
|
|
94
|
+
# ... whatever produces the artifact runs here ...
|
|
95
|
+
precheck settle # run the frozen checks, record PASS / FAIL / TIMEOUT
|
|
96
|
+
precheck audit # mutate the artifacts; find checks that cannot fail
|
|
97
|
+
precheck verify # walk the hash chain; detect edited history
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`settle` exits `2` on any failure. `audit --strict` exits `3` if a check survived
|
|
101
|
+
a mutation. `verify` exits `1` if the chain is broken. All three drop straight
|
|
102
|
+
into CI.
|
|
103
|
+
|
|
104
|
+
### As a GitHub Action
|
|
105
|
+
|
|
106
|
+
```yaml
|
|
107
|
+
jobs:
|
|
108
|
+
verify-the-agent:
|
|
109
|
+
runs-on: ubuntu-latest
|
|
110
|
+
steps:
|
|
111
|
+
- uses: actions/checkout@v4
|
|
112
|
+
- uses: simin-yuan/precheck@main
|
|
113
|
+
with:
|
|
114
|
+
command: audit # settle, then mutate and re-run
|
|
115
|
+
strict: "true" # fail the job if a check could not fail
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Put `precheck register` in the job *before* the step that produces the artifact.
|
|
119
|
+
A check frozen after the fact is reported as `NOT_REGISTERED`, and `verify` fails.
|
|
120
|
+
|
|
121
|
+
### The commitments file
|
|
122
|
+
|
|
123
|
+
```json
|
|
124
|
+
{
|
|
125
|
+
"version": 1,
|
|
126
|
+
"commitments": [
|
|
127
|
+
{
|
|
128
|
+
"id": "C1",
|
|
129
|
+
"statement": "the deployment config is valid and safe to ship",
|
|
130
|
+
"check": "python check_config.py",
|
|
131
|
+
"artifacts": ["config.json"]
|
|
132
|
+
}
|
|
133
|
+
]
|
|
134
|
+
}
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`check` is any shell command whose exit code decides the claim. `artifacts` are
|
|
138
|
+
the files that claim is about — the audit mutates those.
|
|
139
|
+
|
|
140
|
+
### Mutations
|
|
141
|
+
|
|
142
|
+
`drop-line` · `blank-value` · `flip-number` · `truncate`
|
|
143
|
+
|
|
144
|
+
Deterministic given `--seed`, so a survivor is reproducible and can be argued
|
|
145
|
+
about. Binary files are never mutated.
|
|
146
|
+
|
|
147
|
+
## Why not just use pytest?
|
|
148
|
+
|
|
149
|
+
`pytest` asks *is the code right?* `precheck` asks *is your proof right?* They
|
|
150
|
+
are different questions, and the second one currently has no tooling:
|
|
151
|
+
|
|
152
|
+
- `pytest` has no opinion on who wrote the test, or when. `precheck` refuses a
|
|
153
|
+
check that was written after the result it judges.
|
|
154
|
+
- `pytest` cannot tell you that a passing test would also pass on a broken
|
|
155
|
+
file. `precheck` mutates and re-runs to find out.
|
|
156
|
+
- `pytest` results live in your terminal. `precheck` records them in a chain a
|
|
157
|
+
third party can verify without trusting you.
|
|
158
|
+
|
|
159
|
+
You keep using pytest. You point `precheck` at it.
|
|
160
|
+
|
|
161
|
+
## How the tamper-evidence works
|
|
162
|
+
|
|
163
|
+
Each ledger entry is canonical JSON (sorted keys, fixed separators), and holds
|
|
164
|
+
`prev` — the previous entry's sha256 — plus its own hash. Editing any byte of any
|
|
165
|
+
entry re-hashes to something different, and deleting an entry breaks the link
|
|
166
|
+
after it. `verify` re-walks the whole file and reports:
|
|
167
|
+
|
|
168
|
+
| finding | meaning |
|
|
169
|
+
|---|---|
|
|
170
|
+
| `HASH_MISMATCH` | an entry's body was edited |
|
|
171
|
+
| `BROKEN_LINK` | an entry was deleted, or its predecessor was replaced |
|
|
172
|
+
| `SEQ_GAP` | entries were removed from the middle |
|
|
173
|
+
| `COMMITMENTS_MODIFIED` | the check was rewritten after it was frozen |
|
|
174
|
+
| `NOT_REGISTERED` | checks were never frozen at all |
|
|
175
|
+
|
|
176
|
+
Commit `.precheck/` with your code. That is the point — a reader can re-run
|
|
177
|
+
`precheck verify` on your repository and see for themselves.
|
|
178
|
+
|
|
179
|
+
## Limits — read this before you trust it
|
|
180
|
+
|
|
181
|
+
- **A surviving mutation is a question, not a bug.** The mutated line may be
|
|
182
|
+
genuinely irrelevant to the claim. `precheck` will not tell you which; a human
|
|
183
|
+
decides.
|
|
184
|
+
- **Surviving every mutation is not proof that a check is vacuous.** The
|
|
185
|
+
mutations are a fixed sample of four. A check can be useless in ways this
|
|
186
|
+
sample never touches.
|
|
187
|
+
- **The chain proves history was not edited after the fact. It does not prove
|
|
188
|
+
the first entry was honest.** Whoever writes the first `register` entry can
|
|
189
|
+
still write a weak check. Pre-registration raises the cost of gaming the
|
|
190
|
+
result; it does not make gaming impossible.
|
|
191
|
+
- **A weak artifact list defeats it.** If you declare no artifacts, there is
|
|
192
|
+
nothing to mutate and `audit` will honestly report that it examined nothing.
|
|
193
|
+
Declare the files the claim is actually about.
|
|
194
|
+
- **Exit codes only.** `precheck` has no idea what your check *means*. A check
|
|
195
|
+
that prints a lie and exits 0 is a check that passes.
|
|
196
|
+
|
|
197
|
+
## Status
|
|
198
|
+
|
|
199
|
+
Alpha. 19 unit tests over the chain, the freeze, and the audit; a runnable
|
|
200
|
+
end-to-end demo; no dependencies. The audit's mutation set is deliberately small
|
|
201
|
+
and readable — adding mutations that you cannot explain to a reader would make
|
|
202
|
+
the output less trustworthy, not more.
|
|
203
|
+
|
|
204
|
+
## License
|
|
205
|
+
|
|
206
|
+
MIT © 2026 Simin Yuan
|
|
207
|
+
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""precheck command line.
|
|
2
|
+
|
|
3
|
+
precheck init write a commitments template
|
|
4
|
+
precheck register freeze the commitments, before the run
|
|
5
|
+
precheck settle run the frozen checks, record verdicts
|
|
6
|
+
precheck audit mutate artefacts, find checks that can't fail
|
|
7
|
+
precheck verify walk the chain, detect edited history
|
|
8
|
+
precheck status one-screen summary
|
|
9
|
+
"""
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from . import core
|
|
16
|
+
from .core import PrecheckError
|
|
17
|
+
from .runtime import run_check
|
|
18
|
+
from .vacuity import MUTATIONS, audit
|
|
19
|
+
|
|
20
|
+
TEMPLATE = {
|
|
21
|
+
"version": 1,
|
|
22
|
+
"note": ("Write the check BEFORE the work it judges. `check` is a shell "
|
|
23
|
+
"command whose exit code decides the claim; `artifacts` are the "
|
|
24
|
+
"files that claim is about (used by `precheck audit`)."),
|
|
25
|
+
"commitments": [
|
|
26
|
+
{
|
|
27
|
+
"id": "C1",
|
|
28
|
+
"statement": "replace me: what is being claimed",
|
|
29
|
+
"check": "replace me: a command that exits non-zero if it is false",
|
|
30
|
+
"artifacts": ["replace/me.txt"],
|
|
31
|
+
}
|
|
32
|
+
],
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _fail(msg, code=1):
|
|
37
|
+
print(msg, file=sys.stderr)
|
|
38
|
+
return code
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def cmd_init(args):
|
|
42
|
+
p = core.commitments_path(args.root)
|
|
43
|
+
if os.path.exists(p):
|
|
44
|
+
return _fail("%s already exists -- not overwriting" % p)
|
|
45
|
+
core.save_commitments(args.root, TEMPLATE)
|
|
46
|
+
print("wrote %s" % p)
|
|
47
|
+
print("edit it, then: precheck register")
|
|
48
|
+
return 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def cmd_register(args):
|
|
52
|
+
data = core.load_commitments(args.root)
|
|
53
|
+
if not data.get("commitments"):
|
|
54
|
+
return _fail("commitments list is empty -- nothing to freeze")
|
|
55
|
+
h = core.commitments_hash(args.root)
|
|
56
|
+
entry = core.append(args.root, "register",
|
|
57
|
+
{"commitments_sha256": h,
|
|
58
|
+
"count": len(data["commitments"]),
|
|
59
|
+
"ids": [c.get("id") for c in data["commitments"]]})
|
|
60
|
+
print("froze %d commitment(s) at seq=%s sha256=%s"
|
|
61
|
+
% (len(data["commitments"]), entry["seq"], h[:12]))
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cmd_settle(args):
|
|
66
|
+
data = core.load_commitments(args.root)
|
|
67
|
+
results = []
|
|
68
|
+
for c in data.get("commitments", []):
|
|
69
|
+
cid = c.get("id", "?")
|
|
70
|
+
code, out = run_check(c["check"], cwd=args.root, timeout=args.timeout)
|
|
71
|
+
if code is None:
|
|
72
|
+
verdict = "TIMEOUT"
|
|
73
|
+
elif code == 0:
|
|
74
|
+
verdict = "PASS"
|
|
75
|
+
else:
|
|
76
|
+
verdict = "FAIL"
|
|
77
|
+
results.append({"id": cid, "verdict": verdict, "exit": code,
|
|
78
|
+
"statement": c.get("statement", ""),
|
|
79
|
+
"check": c["check"]})
|
|
80
|
+
print("%-5s %s" % (verdict, c.get("statement") or cid))
|
|
81
|
+
if verdict != "PASS" and args.verbose:
|
|
82
|
+
print(" $ %s\n exit=%s\n%s" % (c["check"], code, out.rstrip()))
|
|
83
|
+
|
|
84
|
+
entry = core.append(args.root, "settle", {"results": results})
|
|
85
|
+
bad = [r for r in results if r["verdict"] != "PASS"]
|
|
86
|
+
print("\n%d/%d passed (seq=%s)" % (len(results) - len(bad), len(results),
|
|
87
|
+
entry["seq"]))
|
|
88
|
+
if args.json:
|
|
89
|
+
print(json.dumps(results, ensure_ascii=False, indent=2))
|
|
90
|
+
return 0 if not bad else 2
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def cmd_audit(args):
|
|
94
|
+
data = core.load_commitments(args.root)
|
|
95
|
+
kinds = tuple(k.strip() for k in args.mutations.split(",") if k.strip())
|
|
96
|
+
bad = [k for k in kinds if k not in MUTATIONS]
|
|
97
|
+
if bad:
|
|
98
|
+
return _fail("unknown mutation(s): %s (known: %s)"
|
|
99
|
+
% (", ".join(bad), ", ".join(MUTATIONS)))
|
|
100
|
+
|
|
101
|
+
def on_event(cid, art, kind, code, detail):
|
|
102
|
+
if args.verbose:
|
|
103
|
+
print(" %s %s on %s -> exit=%s [%s]" % (cid, kind, art, code, detail))
|
|
104
|
+
|
|
105
|
+
res = audit(args.root, data, mutations=kinds, seed=args.seed,
|
|
106
|
+
timeout=args.timeout, on_event=on_event)
|
|
107
|
+
entry = core.append(args.root, "audit",
|
|
108
|
+
{"examined": res["examined"],
|
|
109
|
+
"escaped": res["escaped"],
|
|
110
|
+
"skipped": res["skipped"],
|
|
111
|
+
"seed": args.seed,
|
|
112
|
+
"mutations": list(kinds)})
|
|
113
|
+
|
|
114
|
+
print("ran %d mutation(s) across the declared artefacts" % res["examined"])
|
|
115
|
+
for s in res["skipped"]:
|
|
116
|
+
print(" skipped %s (%s: %s)" % (s["artifact"], s["why"], s["id"]))
|
|
117
|
+
if res["escaped"]:
|
|
118
|
+
print("\n%d check/mutation pair(s) survived -- these prove nothing yet:"
|
|
119
|
+
% len(res["escaped"]))
|
|
120
|
+
for e in res["escaped"]:
|
|
121
|
+
print(" ? %s [%s on %s]" % (e["id"], e["mutation"], e["artifact"]))
|
|
122
|
+
print(" %s" % e.get("detail", ""))
|
|
123
|
+
print(" the check still exited 0")
|
|
124
|
+
print("\nThis is a question list, not a bug list. Some survivors are "
|
|
125
|
+
"legitimate.")
|
|
126
|
+
else:
|
|
127
|
+
print("every mutation was caught by its check.")
|
|
128
|
+
print("(seq=%s)" % entry["seq"])
|
|
129
|
+
if args.strict and res["escaped"]:
|
|
130
|
+
return 3
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cmd_verify(args):
|
|
135
|
+
res = core.verify(args.root)
|
|
136
|
+
for f in res["findings"]:
|
|
137
|
+
mark = {"high": "!", "info": "."}.get(f["level"], ".")
|
|
138
|
+
print("%s %s" % (mark, f["message"]))
|
|
139
|
+
print("\n%d %s; chain %s"
|
|
140
|
+
% (res["entries"], "entry" if res["entries"] == 1 else "entries",
|
|
141
|
+
"consistent" if res["ok"] else "BROKEN"))
|
|
142
|
+
return 0 if res["ok"] else 1
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def cmd_status(args):
|
|
146
|
+
try:
|
|
147
|
+
data = core.load_commitments(args.root)
|
|
148
|
+
n = len(data.get("commitments", []))
|
|
149
|
+
except PrecheckError as e:
|
|
150
|
+
return _fail(str(e))
|
|
151
|
+
entries = core.read_ledger(args.root)
|
|
152
|
+
kinds = {}
|
|
153
|
+
for e in entries:
|
|
154
|
+
kinds[e.get("kind")] = kinds.get(e.get("kind"), 0) + 1
|
|
155
|
+
last = entries[-1] if entries else None
|
|
156
|
+
print("commitments : %d" % n)
|
|
157
|
+
print("ledger : %d entries %s" % (len(entries), kinds or ""))
|
|
158
|
+
if last:
|
|
159
|
+
print("head : seq=%s kind=%s %s" % (last.get("seq"),
|
|
160
|
+
last.get("kind"),
|
|
161
|
+
(last.get("hash") or "")[:12]))
|
|
162
|
+
v = core.verify(args.root)
|
|
163
|
+
print("chain : %s" % ("ok" if v["ok"] else "BROKEN"))
|
|
164
|
+
for f in v["findings"]:
|
|
165
|
+
if f["level"] == "high":
|
|
166
|
+
print(" ! %s" % f["message"])
|
|
167
|
+
return 0 if v["ok"] else 1
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def build_parser():
|
|
171
|
+
p = argparse.ArgumentParser(prog="precheck", description=__doc__,
|
|
172
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
173
|
+
p.add_argument("--root", default=".", help="repository root (default: .)")
|
|
174
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
175
|
+
|
|
176
|
+
sub.add_parser("init", help="write a commitments template").set_defaults(
|
|
177
|
+
func=cmd_init)
|
|
178
|
+
sub.add_parser("register", help="freeze commitments before the run"
|
|
179
|
+
).set_defaults(func=cmd_register)
|
|
180
|
+
|
|
181
|
+
s = sub.add_parser("settle", help="run the frozen checks")
|
|
182
|
+
s.add_argument("--timeout", type=int, default=600)
|
|
183
|
+
s.add_argument("--json", action="store_true")
|
|
184
|
+
s.add_argument("-v", "--verbose", action="store_true")
|
|
185
|
+
s.set_defaults(func=cmd_settle)
|
|
186
|
+
|
|
187
|
+
a = sub.add_parser("audit", help="find checks that cannot fail")
|
|
188
|
+
a.add_argument("--mutations", default=",".join(MUTATIONS))
|
|
189
|
+
a.add_argument("--seed", type=int, default=0)
|
|
190
|
+
a.add_argument("--timeout", type=int, default=600)
|
|
191
|
+
a.add_argument("--strict", action="store_true",
|
|
192
|
+
help="exit non-zero if any mutation survived (for CI)")
|
|
193
|
+
a.add_argument("-v", "--verbose", action="store_true")
|
|
194
|
+
a.set_defaults(func=cmd_audit)
|
|
195
|
+
|
|
196
|
+
sub.add_parser("verify", help="walk the chain").set_defaults(func=cmd_verify)
|
|
197
|
+
sub.add_parser("status", help="summary").set_defaults(func=cmd_status)
|
|
198
|
+
return p
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def main(argv=None):
|
|
202
|
+
args = build_parser().parse_args(argv)
|
|
203
|
+
try:
|
|
204
|
+
return args.func(args)
|
|
205
|
+
except PrecheckError as e:
|
|
206
|
+
return _fail("precheck: %s" % e)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
sys.exit(main())
|