proofbundle 3.0.0__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proofbundle-3.0.0/src/proofbundle.egg-info → proofbundle-3.0.1}/PKG-INFO +61 -42
- {proofbundle-3.0.0 → proofbundle-3.0.1}/README.md +60 -41
- {proofbundle-3.0.0 → proofbundle-3.0.1}/pyproject.toml +1 -1
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/__init__.py +1 -1
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/eee.py +17 -4
- {proofbundle-3.0.0 → proofbundle-3.0.1/src/proofbundle.egg-info}/PKG-INFO +61 -42
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle.egg-info/SOURCES.txt +1 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_hf_entry_verify_eee_digest.py +8 -0
- proofbundle-3.0.1/tests/test_release_integrity_gate.py +88 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/LICENSE +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/setup.cfg +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/_inspect_registry.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/_integration.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/_strict_json.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/__init__.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/_provenance.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/inspect_ai.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/lm_eval.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/promptfoo.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/adapters/samples.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors_chia.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors_chia_add.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors_markovian.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors_ots.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/anchors_rfc3161.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/beacon.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/bundle.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/canonical.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/checkpoint.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/cli.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/decision.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/demo.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/dsse.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/eee_eval_schema.json +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/emit.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/errors.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/evalclaim.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/experimental/__init__.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/experimental/enclave.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/hf_evals.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/inspect_hook.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/intoto.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/kbjwt.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/merkle.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/persample.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/policy.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/prereg.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/py.typed +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/pytest_plugin.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/sdjwt.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/sdjwt_issue.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/signature.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/statuslist.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle/tlogproof.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle.egg-info/dependency_links.txt +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle.egg-info/entry_points.txt +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle.egg-info/requires.txt +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/src/proofbundle.egg-info/top_level.txt +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_adapters.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_adversarial.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchor_target_trustedtime.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_chia.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_chia_add.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_chia_claims.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_generic.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_markovian.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_ots.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_anchors_rfc3161.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_beacon.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_branch_base_check.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_bundle.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_bundle_robustness.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_canonical.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_checkpoint.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_claims_hygiene.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_cli.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_cli_eval.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_cli_require_anchor.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_conformance.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_cosignature.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_cosignature_mldsa.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_anchors.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_cli.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_evidence.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_fuzz.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_hardening.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_policy.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_schema.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_validator_api.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_decision_verify.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_demo.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_docs_truth.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_dup_key_reject.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_ed25519_semantics.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_eee.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_emit.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_eval_claim_schema.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_evalclaim.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_examples.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_experimental_enclave.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_fork_pr_secret_isolation.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_fuzz_parsers.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_hf_evals.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_inspect_hook.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_claims_hygiene.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_content_root_migration.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_dsse.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_eval_result.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_examples.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_spec_diff.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_intoto_svr.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_kbjwt.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_merkle.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_merkle_property.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_ok_semantics.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_oversized_int_guard.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_persample.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_pinned_key_validation.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_policy_explain_lint.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_predicate_type_enforcement.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_prereg.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_promptfoo.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_provenance.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_pytest_plugin.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_rekor_interop.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_rfc6962_external_vectors.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_schema.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_sdjwt_duplicate_cnf.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_sdjwt_issue.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_sdjwt_reference.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_sdjwt_verify_binding.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_signature.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_statuslist.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_tlogproof.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_trust_policy.py +0 -0
- {proofbundle-3.0.0 → proofbundle-3.0.1}/tests/test_verify_matrix.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proofbundle
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
|
|
5
5
|
Author: Konrad Gruszka
|
|
6
6
|
License: MIT
|
|
@@ -75,12 +75,28 @@ Merkle, one file, no server, no network.
|
|
|
75
75
|
[](https://doi.org/10.5281/zenodo.21230467)
|
|
76
76
|
[](https://github.com/astral-sh/ruff)
|
|
77
77
|
[](https://github.com/b7n0de/proofbundle/blob/main/scripts/mutation_check.py)
|
|
78
|
-
|
|
78
|
+
[](https://pypi.org/project/proofbundle/#files)
|
|
79
|
+
[](https://slsa.dev/spec/v1.0/provenance)
|
|
79
80
|
|
|
80
81
|
**Reviewing this for adoption?** Start with the 30-minute adversarial audit path: **[docs/REVIEWERS.md](https://github.com/b7n0de/proofbundle/blob/main/docs/REVIEWERS.md)**.
|
|
81
82
|
|
|
82
83
|
</div>
|
|
83
84
|
|
|
85
|
+
## Contents
|
|
86
|
+
|
|
87
|
+
- [60-second try](#60-second-try-offline)
|
|
88
|
+
- [Inspect-native?](#inspect-native-metr-task-standard-uk-aisi-ecosystem)
|
|
89
|
+
- [The problem](#the-problem)
|
|
90
|
+
- [What a receipt proves, and what it doesn't](#what-a-receipt-proves-and-what-it-doesnt)
|
|
91
|
+
- [How it fits together](#how-it-fits-together)
|
|
92
|
+
- [Post-quantum posture](#post-quantum-posture-honest-two-layers)
|
|
93
|
+
- [What's in the box](#whats-in-the-box)
|
|
94
|
+
- [Docs](#docs)
|
|
95
|
+
- [Install](#install)
|
|
96
|
+
- [Status, scope and roadmap](#status-scope-and-roadmap)
|
|
97
|
+
- [Contributing](#contributing)
|
|
98
|
+
- [License](#license)
|
|
99
|
+
|
|
84
100
|
## 60-second try (offline)
|
|
85
101
|
|
|
86
102
|
```bash
|
|
@@ -91,21 +107,9 @@ git clone https://github.com/b7n0de/proofbundle && cd proofbundle
|
|
|
91
107
|
pip install -e ".[eval,inspect]" && make demo # or `make full-demo` for log -> receipt -> verify
|
|
92
108
|
```
|
|
93
109
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
98
|
-
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
99
|
-
|
|
100
|
-
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
101
|
-
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
102
|
-
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
103
|
-
|
|
104
|
-
## What the demo shows
|
|
105
|
-
|
|
106
|
-
You'll see an honest receipt verify `=> OK`, then six independent tampers each verify `FAILED`, then
|
|
107
|
-
a swapped sample get caught — all in memory. `proofbundle demo` exits non-zero if any tamper slips through,
|
|
108
|
-
so it's also a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
110
|
+
The demo runs entirely in memory: an honest receipt verifies `=> OK`, six independent tampers each
|
|
111
|
+
verify `FAILED`, and a swapped sample gets caught. `proofbundle demo` exits non-zero if any tamper
|
|
112
|
+
slips through, so it doubles as a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
109
113
|
|
|
110
114
|
```bash
|
|
111
115
|
# verify a real hosted receipt without writing any code:
|
|
@@ -120,7 +124,7 @@ proofbundle verify receipt.json # exit 0 = crypto OK, 1 = crypto/verifica
|
|
|
120
124
|
proofbundle verify receipt.json --policy trust_policy.json # POLICY: OK | FAIL (exit 3) | NOT_EVALUATED
|
|
121
125
|
```
|
|
122
126
|
|
|
123
|
-
## Inspect-native? (METR Task Standard
|
|
127
|
+
## Inspect-native? (METR Task Standard, UK-AISI ecosystem)
|
|
124
128
|
|
|
125
129
|
The receipt layer runs directly on [Inspect AI](https://inspect.aisi.org.uk/) — and the proof is
|
|
126
130
|
reproducible offline in minutes:
|
|
@@ -135,7 +139,22 @@ In your own pipeline the end-of-task hook signs every run automatically. Walkthr
|
|
|
135
139
|
**[docs/INSPECT_HAPPY_PATH.md](https://github.com/b7n0de/proofbundle/blob/main/docs/INSPECT_HAPPY_PATH.md)** · worked example:
|
|
136
140
|
**[examples/inspect_receipt.py](https://github.com/b7n0de/proofbundle/blob/main/examples/inspect_receipt.py)**.
|
|
137
141
|
|
|
138
|
-
##
|
|
142
|
+
## The problem
|
|
143
|
+
|
|
144
|
+
Every AI eval number you read — a safety benchmark, a capability score, a leaderboard entry — is an
|
|
145
|
+
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
146
|
+
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
147
|
+
|
|
148
|
+
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
149
|
+
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
150
|
+
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
151
|
+
|
|
152
|
+
In plain language: a proofbundle receipt is the cash-register receipt of an AI test result. It shows
|
|
153
|
+
who claimed the number and that nobody quietly changed it afterwards. It does not show the test was
|
|
154
|
+
good — the way a cash-register receipt does not show the meal was good — but without a receipt there
|
|
155
|
+
is nothing to check at all.
|
|
156
|
+
|
|
157
|
+
## What a receipt proves, and what it doesn't
|
|
139
158
|
|
|
140
159
|
| ✅ It proves | ❌ It does **not** prove |
|
|
141
160
|
|---|---|
|
|
@@ -149,6 +168,21 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
|
|
|
149
168
|
and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
|
|
150
169
|
**[THREAT_MODEL.md](https://github.com/b7n0de/proofbundle/blob/main/THREAT_MODEL.md)**.
|
|
151
170
|
|
|
171
|
+
## How it fits together
|
|
172
|
+
|
|
173
|
+
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
174
|
+
|
|
175
|
+
```mermaid
|
|
176
|
+
flowchart LR
|
|
177
|
+
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
178
|
+
A --> R["receipt<br/>one portable file"]
|
|
179
|
+
R --> V{{"proofbundle verify — offline"}}
|
|
180
|
+
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
181
|
+
C --> OK(["CRYPTO: OK / FAILED"])
|
|
182
|
+
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
183
|
+
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
184
|
+
```
|
|
185
|
+
|
|
152
186
|
## Post-quantum posture (honest, two layers)
|
|
153
187
|
|
|
154
188
|
proofbundle is **not** "quantum-proof" or "quantum-safe" as a whole. It combines two cryptographic layers
|
|
@@ -173,28 +207,6 @@ On the witness side, C2SP checkpoints already carry post-quantum **ML-DSA-44** (
|
|
|
173
207
|
(`proofbundle[pq]`); a post-quantum *payload* signature — crypto-agility for the receipt itself — is on the
|
|
174
208
|
roadmap.
|
|
175
209
|
|
|
176
|
-
## In plain language
|
|
177
|
-
|
|
178
|
-
A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
|
|
179
|
-
number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
|
|
180
|
-
cash-register receipt does not show the meal was good — but without a receipt there is nothing to
|
|
181
|
-
check at all.
|
|
182
|
-
|
|
183
|
-
## How it fits together
|
|
184
|
-
|
|
185
|
-
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
186
|
-
|
|
187
|
-
```mermaid
|
|
188
|
-
flowchart LR
|
|
189
|
-
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
190
|
-
A --> R["receipt<br/>one portable file"]
|
|
191
|
-
R --> V{{"proofbundle verify — offline"}}
|
|
192
|
-
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
193
|
-
C --> OK(["CRYPTO: OK / FAILED"])
|
|
194
|
-
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
195
|
-
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
196
|
-
```
|
|
197
|
-
|
|
198
210
|
## What's in the box
|
|
199
211
|
|
|
200
212
|
- **Core** — Ed25519 signature + RFC 6962 / 9162 Merkle inclusion, verified fully offline. Checks a
|
|
@@ -285,7 +297,7 @@ pip install "proofbundle[pq]" # verify ML-DSA-44 (post-quantum) witnes
|
|
|
285
297
|
Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 comes from
|
|
286
298
|
`cryptography`; Merkle hashing is RFC 6962.
|
|
287
299
|
|
|
288
|
-
## Status
|
|
300
|
+
## Status, scope and roadmap
|
|
289
301
|
|
|
290
302
|
Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
|
|
291
303
|
is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
|
|
@@ -293,6 +305,13 @@ is anchored to external RFC 6962 vectors and a real Rekor proof, not just its ow
|
|
|
293
305
|
by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
|
|
294
306
|
[SECURITY.md](https://github.com/b7n0de/proofbundle/blob/main/SECURITY.md).
|
|
295
307
|
|
|
308
|
+
**Roadmap (stated honestly, all forward-looking):**
|
|
309
|
+
|
|
310
|
+
- A post-quantum *payload* signature (crypto-agility for the receipt itself) is on the roadmap; today the
|
|
311
|
+
post-quantum coverage is witness-side ML-DSA-44 cosignatures only.
|
|
312
|
+
- A CLI flag to select the content-root algorithm is still deferred (`jcs-sha256-v1` is the signed default).
|
|
313
|
+
- **Preview:** a TEE-attestation bridge — see [docs/EXPERIMENTAL_ENCLAVE.md](https://github.com/b7n0de/proofbundle/blob/main/docs/EXPERIMENTAL_ENCLAVE.md).
|
|
314
|
+
|
|
296
315
|
## Contributing
|
|
297
316
|
|
|
298
317
|
See [CONTRIBUTING.md](https://github.com/b7n0de/proofbundle/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/b7n0de/proofbundle/blob/main/CODE_OF_CONDUCT.md). Good first
|
|
@@ -21,12 +21,28 @@ Merkle, one file, no server, no network.
|
|
|
21
21
|
[](https://doi.org/10.5281/zenodo.21230467)
|
|
22
22
|
[](https://github.com/astral-sh/ruff)
|
|
23
23
|
[](https://github.com/b7n0de/proofbundle/blob/main/scripts/mutation_check.py)
|
|
24
|
-
|
|
24
|
+
[](https://pypi.org/project/proofbundle/#files)
|
|
25
|
+
[](https://slsa.dev/spec/v1.0/provenance)
|
|
25
26
|
|
|
26
27
|
**Reviewing this for adoption?** Start with the 30-minute adversarial audit path: **[docs/REVIEWERS.md](https://github.com/b7n0de/proofbundle/blob/main/docs/REVIEWERS.md)**.
|
|
27
28
|
|
|
28
29
|
</div>
|
|
29
30
|
|
|
31
|
+
## Contents
|
|
32
|
+
|
|
33
|
+
- [60-second try](#60-second-try-offline)
|
|
34
|
+
- [Inspect-native?](#inspect-native-metr-task-standard-uk-aisi-ecosystem)
|
|
35
|
+
- [The problem](#the-problem)
|
|
36
|
+
- [What a receipt proves, and what it doesn't](#what-a-receipt-proves-and-what-it-doesnt)
|
|
37
|
+
- [How it fits together](#how-it-fits-together)
|
|
38
|
+
- [Post-quantum posture](#post-quantum-posture-honest-two-layers)
|
|
39
|
+
- [What's in the box](#whats-in-the-box)
|
|
40
|
+
- [Docs](#docs)
|
|
41
|
+
- [Install](#install)
|
|
42
|
+
- [Status, scope and roadmap](#status-scope-and-roadmap)
|
|
43
|
+
- [Contributing](#contributing)
|
|
44
|
+
- [License](#license)
|
|
45
|
+
|
|
30
46
|
## 60-second try (offline)
|
|
31
47
|
|
|
32
48
|
```bash
|
|
@@ -37,21 +53,9 @@ git clone https://github.com/b7n0de/proofbundle && cd proofbundle
|
|
|
37
53
|
pip install -e ".[eval,inspect]" && make demo # or `make full-demo` for log -> receipt -> verify
|
|
38
54
|
```
|
|
39
55
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
44
|
-
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
45
|
-
|
|
46
|
-
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
47
|
-
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
48
|
-
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
49
|
-
|
|
50
|
-
## What the demo shows
|
|
51
|
-
|
|
52
|
-
You'll see an honest receipt verify `=> OK`, then six independent tampers each verify `FAILED`, then
|
|
53
|
-
a swapped sample get caught — all in memory. `proofbundle demo` exits non-zero if any tamper slips through,
|
|
54
|
-
so it's also a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
56
|
+
The demo runs entirely in memory: an honest receipt verifies `=> OK`, six independent tampers each
|
|
57
|
+
verify `FAILED`, and a swapped sample gets caught. `proofbundle demo` exits non-zero if any tamper
|
|
58
|
+
slips through, so it doubles as a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
55
59
|
|
|
56
60
|
```bash
|
|
57
61
|
# verify a real hosted receipt without writing any code:
|
|
@@ -66,7 +70,7 @@ proofbundle verify receipt.json # exit 0 = crypto OK, 1 = crypto/verifica
|
|
|
66
70
|
proofbundle verify receipt.json --policy trust_policy.json # POLICY: OK | FAIL (exit 3) | NOT_EVALUATED
|
|
67
71
|
```
|
|
68
72
|
|
|
69
|
-
## Inspect-native? (METR Task Standard
|
|
73
|
+
## Inspect-native? (METR Task Standard, UK-AISI ecosystem)
|
|
70
74
|
|
|
71
75
|
The receipt layer runs directly on [Inspect AI](https://inspect.aisi.org.uk/) — and the proof is
|
|
72
76
|
reproducible offline in minutes:
|
|
@@ -81,7 +85,22 @@ In your own pipeline the end-of-task hook signs every run automatically. Walkthr
|
|
|
81
85
|
**[docs/INSPECT_HAPPY_PATH.md](https://github.com/b7n0de/proofbundle/blob/main/docs/INSPECT_HAPPY_PATH.md)** · worked example:
|
|
82
86
|
**[examples/inspect_receipt.py](https://github.com/b7n0de/proofbundle/blob/main/examples/inspect_receipt.py)**.
|
|
83
87
|
|
|
84
|
-
##
|
|
88
|
+
## The problem
|
|
89
|
+
|
|
90
|
+
Every AI eval number you read — a safety benchmark, a capability score, a leaderboard entry — is an
|
|
91
|
+
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
92
|
+
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
93
|
+
|
|
94
|
+
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
95
|
+
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
96
|
+
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
97
|
+
|
|
98
|
+
In plain language: a proofbundle receipt is the cash-register receipt of an AI test result. It shows
|
|
99
|
+
who claimed the number and that nobody quietly changed it afterwards. It does not show the test was
|
|
100
|
+
good — the way a cash-register receipt does not show the meal was good — but without a receipt there
|
|
101
|
+
is nothing to check at all.
|
|
102
|
+
|
|
103
|
+
## What a receipt proves, and what it doesn't
|
|
85
104
|
|
|
86
105
|
| ✅ It proves | ❌ It does **not** prove |
|
|
87
106
|
|---|---|
|
|
@@ -95,6 +114,21 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
|
|
|
95
114
|
and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
|
|
96
115
|
**[THREAT_MODEL.md](https://github.com/b7n0de/proofbundle/blob/main/THREAT_MODEL.md)**.
|
|
97
116
|
|
|
117
|
+
## How it fits together
|
|
118
|
+
|
|
119
|
+
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
120
|
+
|
|
121
|
+
```mermaid
|
|
122
|
+
flowchart LR
|
|
123
|
+
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
124
|
+
A --> R["receipt<br/>one portable file"]
|
|
125
|
+
R --> V{{"proofbundle verify — offline"}}
|
|
126
|
+
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
127
|
+
C --> OK(["CRYPTO: OK / FAILED"])
|
|
128
|
+
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
129
|
+
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
130
|
+
```
|
|
131
|
+
|
|
98
132
|
## Post-quantum posture (honest, two layers)
|
|
99
133
|
|
|
100
134
|
proofbundle is **not** "quantum-proof" or "quantum-safe" as a whole. It combines two cryptographic layers
|
|
@@ -119,28 +153,6 @@ On the witness side, C2SP checkpoints already carry post-quantum **ML-DSA-44** (
|
|
|
119
153
|
(`proofbundle[pq]`); a post-quantum *payload* signature — crypto-agility for the receipt itself — is on the
|
|
120
154
|
roadmap.
|
|
121
155
|
|
|
122
|
-
## In plain language
|
|
123
|
-
|
|
124
|
-
A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
|
|
125
|
-
number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
|
|
126
|
-
cash-register receipt does not show the meal was good — but without a receipt there is nothing to
|
|
127
|
-
check at all.
|
|
128
|
-
|
|
129
|
-
## How it fits together
|
|
130
|
-
|
|
131
|
-
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
132
|
-
|
|
133
|
-
```mermaid
|
|
134
|
-
flowchart LR
|
|
135
|
-
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
136
|
-
A --> R["receipt<br/>one portable file"]
|
|
137
|
-
R --> V{{"proofbundle verify — offline"}}
|
|
138
|
-
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
139
|
-
C --> OK(["CRYPTO: OK / FAILED"])
|
|
140
|
-
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
141
|
-
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
142
|
-
```
|
|
143
|
-
|
|
144
156
|
## What's in the box
|
|
145
157
|
|
|
146
158
|
- **Core** — Ed25519 signature + RFC 6962 / 9162 Merkle inclusion, verified fully offline. Checks a
|
|
@@ -231,7 +243,7 @@ pip install "proofbundle[pq]" # verify ML-DSA-44 (post-quantum) witnes
|
|
|
231
243
|
Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 comes from
|
|
232
244
|
`cryptography`; Merkle hashing is RFC 6962.
|
|
233
245
|
|
|
234
|
-
## Status
|
|
246
|
+
## Status, scope and roadmap
|
|
235
247
|
|
|
236
248
|
Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
|
|
237
249
|
is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
|
|
@@ -239,6 +251,13 @@ is anchored to external RFC 6962 vectors and a real Rekor proof, not just its ow
|
|
|
239
251
|
by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
|
|
240
252
|
[SECURITY.md](https://github.com/b7n0de/proofbundle/blob/main/SECURITY.md).
|
|
241
253
|
|
|
254
|
+
**Roadmap (stated honestly, all forward-looking):**
|
|
255
|
+
|
|
256
|
+
- A post-quantum *payload* signature (crypto-agility for the receipt itself) is on the roadmap; today the
|
|
257
|
+
post-quantum coverage is witness-side ML-DSA-44 cosignatures only.
|
|
258
|
+
- A CLI flag to select the content-root algorithm is still deferred (`jcs-sha256-v1` is the signed default).
|
|
259
|
+
- **Preview:** a TEE-attestation bridge — see [docs/EXPERIMENTAL_ENCLAVE.md](https://github.com/b7n0de/proofbundle/blob/main/docs/EXPERIMENTAL_ENCLAVE.md).
|
|
260
|
+
|
|
242
261
|
## Contributing
|
|
243
262
|
|
|
244
263
|
See [CONTRIBUTING.md](https://github.com/b7n0de/proofbundle/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/b7n0de/proofbundle/blob/main/CODE_OF_CONDUCT.md). Good first
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "proofbundle"
|
|
7
|
-
version = "3.0.
|
|
7
|
+
version = "3.0.1"
|
|
8
8
|
description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from typing import TYPE_CHECKING
|
|
15
15
|
|
|
16
|
-
__version__ = "3.0.
|
|
16
|
+
__version__ = "3.0.1"
|
|
17
17
|
|
|
18
18
|
# The `proofbundle/v0.1` normative spec revision this build implements — kept in sync with the
|
|
19
19
|
# `Revision:` line at the top of SPEC.md by tests/test_docs_truth.py (WP-B1, closes #28). Bump
|
|
@@ -99,14 +99,27 @@ def _extract_score(score_details: dict, metric_config: dict) -> str:
|
|
|
99
99
|
|
|
100
100
|
def _model_id_stripped(record: dict) -> dict:
|
|
101
101
|
"""A deep copy of the EEE record with the cleartext model identity removed (WP-I3 privacy fix).
|
|
102
|
-
Removes ``model_info.id`` and the top-level ``evaluation_id``
|
|
103
|
-
over the result cannot be used as a model-id
|
|
104
|
-
binding every score, timestamp, dataset and metric
|
|
102
|
+
Removes ``model_info.id`` and the id-bearing top-level fields (``evaluation_id`` which embeds the
|
|
103
|
+
id, and ``evaluation_result_id``) so a digest over the result cannot be used as a model-id
|
|
104
|
+
confirmation / enumeration oracle, while still binding every score, timestamp, dataset and metric
|
|
105
|
+
for tamper-evidence.
|
|
106
|
+
|
|
107
|
+
M2 (6-lens review 2026-07-11): ``evaluation_result_id`` was left IN the digest record while the
|
|
108
|
+
run_id provenance path already `_leaks_model_id`-guards it — an asymmetry that kept the digest a
|
|
109
|
+
model-id oracle (the id can correlate to / embed the model). It is a provenance identifier, not a
|
|
110
|
+
scored/content field, so stripping it from the DIGEST removes the oracle without weakening the
|
|
111
|
+
tamper-evidence over the actual result content (it stays available for run_id provenance, which
|
|
112
|
+
applies its own leak guard)."""
|
|
105
113
|
import copy # noqa: PLC0415
|
|
106
114
|
r = copy.deepcopy(record)
|
|
107
115
|
if isinstance(r.get("model_info"), dict):
|
|
108
116
|
r["model_info"].pop("id", None)
|
|
109
|
-
r.pop("evaluation_id", None)
|
|
117
|
+
r.pop("evaluation_id", None) # format eval_name/model_id/timestamp — embeds the id
|
|
118
|
+
# M2: strip the per-result ``evaluation_result_id`` (nested in each ``evaluation_results[*]``) —
|
|
119
|
+
# it is a provenance id that can embed/correlate the model id (a digest over it is an oracle).
|
|
120
|
+
for er in r.get("evaluation_results", []) or []:
|
|
121
|
+
if isinstance(er, dict):
|
|
122
|
+
er.pop("evaluation_result_id", None)
|
|
110
123
|
return r
|
|
111
124
|
|
|
112
125
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proofbundle
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
|
|
5
5
|
Author: Konrad Gruszka
|
|
6
6
|
License: MIT
|
|
@@ -75,12 +75,28 @@ Merkle, one file, no server, no network.
|
|
|
75
75
|
[](https://doi.org/10.5281/zenodo.21230467)
|
|
76
76
|
[](https://github.com/astral-sh/ruff)
|
|
77
77
|
[](https://github.com/b7n0de/proofbundle/blob/main/scripts/mutation_check.py)
|
|
78
|
-
|
|
78
|
+
[](https://pypi.org/project/proofbundle/#files)
|
|
79
|
+
[](https://slsa.dev/spec/v1.0/provenance)
|
|
79
80
|
|
|
80
81
|
**Reviewing this for adoption?** Start with the 30-minute adversarial audit path: **[docs/REVIEWERS.md](https://github.com/b7n0de/proofbundle/blob/main/docs/REVIEWERS.md)**.
|
|
81
82
|
|
|
82
83
|
</div>
|
|
83
84
|
|
|
85
|
+
## Contents
|
|
86
|
+
|
|
87
|
+
- [60-second try](#60-second-try-offline)
|
|
88
|
+
- [Inspect-native?](#inspect-native-metr-task-standard-uk-aisi-ecosystem)
|
|
89
|
+
- [The problem](#the-problem)
|
|
90
|
+
- [What a receipt proves, and what it doesn't](#what-a-receipt-proves-and-what-it-doesnt)
|
|
91
|
+
- [How it fits together](#how-it-fits-together)
|
|
92
|
+
- [Post-quantum posture](#post-quantum-posture-honest-two-layers)
|
|
93
|
+
- [What's in the box](#whats-in-the-box)
|
|
94
|
+
- [Docs](#docs)
|
|
95
|
+
- [Install](#install)
|
|
96
|
+
- [Status, scope and roadmap](#status-scope-and-roadmap)
|
|
97
|
+
- [Contributing](#contributing)
|
|
98
|
+
- [License](#license)
|
|
99
|
+
|
|
84
100
|
## 60-second try (offline)
|
|
85
101
|
|
|
86
102
|
```bash
|
|
@@ -91,21 +107,9 @@ git clone https://github.com/b7n0de/proofbundle && cd proofbundle
|
|
|
91
107
|
pip install -e ".[eval,inspect]" && make demo # or `make full-demo` for log -> receipt -> verify
|
|
92
108
|
```
|
|
93
109
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
98
|
-
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
99
|
-
|
|
100
|
-
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
101
|
-
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
102
|
-
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
103
|
-
|
|
104
|
-
## What the demo shows
|
|
105
|
-
|
|
106
|
-
You'll see an honest receipt verify `=> OK`, then six independent tampers each verify `FAILED`, then
|
|
107
|
-
a swapped sample get caught — all in memory. `proofbundle demo` exits non-zero if any tamper slips through,
|
|
108
|
-
so it's also a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
110
|
+
The demo runs entirely in memory: an honest receipt verifies `=> OK`, six independent tampers each
|
|
111
|
+
verify `FAILED`, and a swapped sample gets caught. `proofbundle demo` exits non-zero if any tamper
|
|
112
|
+
slips through, so it doubles as a self-test. Full walkthrough: **[docs/DEMO.md](https://github.com/b7n0de/proofbundle/blob/main/docs/DEMO.md)**.
|
|
109
113
|
|
|
110
114
|
```bash
|
|
111
115
|
# verify a real hosted receipt without writing any code:
|
|
@@ -120,7 +124,7 @@ proofbundle verify receipt.json # exit 0 = crypto OK, 1 = crypto/verifica
|
|
|
120
124
|
proofbundle verify receipt.json --policy trust_policy.json # POLICY: OK | FAIL (exit 3) | NOT_EVALUATED
|
|
121
125
|
```
|
|
122
126
|
|
|
123
|
-
## Inspect-native? (METR Task Standard
|
|
127
|
+
## Inspect-native? (METR Task Standard, UK-AISI ecosystem)
|
|
124
128
|
|
|
125
129
|
The receipt layer runs directly on [Inspect AI](https://inspect.aisi.org.uk/) — and the proof is
|
|
126
130
|
reproducible offline in minutes:
|
|
@@ -135,7 +139,22 @@ In your own pipeline the end-of-task hook signs every run automatically. Walkthr
|
|
|
135
139
|
**[docs/INSPECT_HAPPY_PATH.md](https://github.com/b7n0de/proofbundle/blob/main/docs/INSPECT_HAPPY_PATH.md)** · worked example:
|
|
136
140
|
**[examples/inspect_receipt.py](https://github.com/b7n0de/proofbundle/blob/main/examples/inspect_receipt.py)**.
|
|
137
141
|
|
|
138
|
-
##
|
|
142
|
+
## The problem
|
|
143
|
+
|
|
144
|
+
Every AI eval number you read — a safety benchmark, a capability score, a leaderboard entry — is an
|
|
145
|
+
**unverifiable claim**. You trust the lab. There's no portable way to check, offline, that a result
|
|
146
|
+
was signed by a stated party, hasn't been altered, and covers the samples it claims.
|
|
147
|
+
|
|
148
|
+
proofbundle is that check. It's a small MIT-licensed Python tool (a compact, auditable trusted core,
|
|
149
|
+
depends only on [`cryptography`](https://cryptography.io)) that turns a result into a signed
|
|
150
|
+
receipt anyone can verify from a single file — and it's honest about the line it does not cross.
|
|
151
|
+
|
|
152
|
+
In plain language: a proofbundle receipt is the cash-register receipt of an AI test result. It shows
|
|
153
|
+
who claimed the number and that nobody quietly changed it afterwards. It does not show the test was
|
|
154
|
+
good — the way a cash-register receipt does not show the meal was good — but without a receipt there
|
|
155
|
+
is nothing to check at all.
|
|
156
|
+
|
|
157
|
+
## What a receipt proves, and what it doesn't
|
|
139
158
|
|
|
140
159
|
| ✅ It proves | ❌ It does **not** prove |
|
|
141
160
|
|---|---|
|
|
@@ -149,6 +168,21 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
|
|
|
149
168
|
and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
|
|
150
169
|
**[THREAT_MODEL.md](https://github.com/b7n0de/proofbundle/blob/main/THREAT_MODEL.md)**.
|
|
151
170
|
|
|
171
|
+
## How it fits together
|
|
172
|
+
|
|
173
|
+
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
174
|
+
|
|
175
|
+
```mermaid
|
|
176
|
+
flowchart LR
|
|
177
|
+
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
178
|
+
A --> R["receipt<br/>one portable file"]
|
|
179
|
+
R --> V{{"proofbundle verify — offline"}}
|
|
180
|
+
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
181
|
+
C --> OK(["CRYPTO: OK / FAILED"])
|
|
182
|
+
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
183
|
+
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
184
|
+
```
|
|
185
|
+
|
|
152
186
|
## Post-quantum posture (honest, two layers)
|
|
153
187
|
|
|
154
188
|
proofbundle is **not** "quantum-proof" or "quantum-safe" as a whole. It combines two cryptographic layers
|
|
@@ -173,28 +207,6 @@ On the witness side, C2SP checkpoints already carry post-quantum **ML-DSA-44** (
|
|
|
173
207
|
(`proofbundle[pq]`); a post-quantum *payload* signature — crypto-agility for the receipt itself — is on the
|
|
174
208
|
roadmap.
|
|
175
209
|
|
|
176
|
-
## In plain language
|
|
177
|
-
|
|
178
|
-
A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
|
|
179
|
-
number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
|
|
180
|
-
cash-register receipt does not show the meal was good — but without a receipt there is nothing to
|
|
181
|
-
check at all.
|
|
182
|
-
|
|
183
|
-
## How it fits together
|
|
184
|
-
|
|
185
|
-
*(diagram renders on GitHub — [view it there](https://github.com/b7n0de/proofbundle#how-it-fits-together); PyPI shows the source)*
|
|
186
|
-
|
|
187
|
-
```mermaid
|
|
188
|
-
flowchart LR
|
|
189
|
-
H["eval harness<br/>inspect_ai · lm-eval · promptfoo · pytest"] --> A["adapter → signed claim<br/>salted commitments · provenance · samples root"]
|
|
190
|
-
A --> R["receipt<br/>one portable file"]
|
|
191
|
-
R --> V{{"proofbundle verify — offline"}}
|
|
192
|
-
V --> C["signature · Merkle inclusion · SD-JWT/KB ·<br/>witness quorum · status list · sample openings"]
|
|
193
|
-
C --> OK(["CRYPTO: OK / FAILED"])
|
|
194
|
-
style V fill:#D6248A,stroke:#D6248A,color:#fff
|
|
195
|
-
style OK fill:#D6248A,stroke:#D6248A,color:#fff
|
|
196
|
-
```
|
|
197
|
-
|
|
198
210
|
## What's in the box
|
|
199
211
|
|
|
200
212
|
- **Core** — Ed25519 signature + RFC 6962 / 9162 Merkle inclusion, verified fully offline. Checks a
|
|
@@ -285,7 +297,7 @@ pip install "proofbundle[pq]" # verify ML-DSA-44 (post-quantum) witnes
|
|
|
285
297
|
Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 comes from
|
|
286
298
|
`cryptography`; Merkle hashing is RFC 6962.
|
|
287
299
|
|
|
288
|
-
## Status
|
|
300
|
+
## Status, scope and roadmap
|
|
289
301
|
|
|
290
302
|
Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
|
|
291
303
|
is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
|
|
@@ -293,6 +305,13 @@ is anchored to external RFC 6962 vectors and a real Rekor proof, not just its ow
|
|
|
293
305
|
by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
|
|
294
306
|
[SECURITY.md](https://github.com/b7n0de/proofbundle/blob/main/SECURITY.md).
|
|
295
307
|
|
|
308
|
+
**Roadmap (stated honestly, all forward-looking):**
|
|
309
|
+
|
|
310
|
+
- A post-quantum *payload* signature (crypto-agility for the receipt itself) is on the roadmap; today the
|
|
311
|
+
post-quantum coverage is witness-side ML-DSA-44 cosignatures only.
|
|
312
|
+
- A CLI flag to select the content-root algorithm is still deferred (`jcs-sha256-v1` is the signed default).
|
|
313
|
+
- **Preview:** a TEE-attestation bridge — see [docs/EXPERIMENTAL_ENCLAVE.md](https://github.com/b7n0de/proofbundle/blob/main/docs/EXPERIMENTAL_ENCLAVE.md).
|
|
314
|
+
|
|
296
315
|
## Contributing
|
|
297
316
|
|
|
298
317
|
See [CONTRIBUTING.md](https://github.com/b7n0de/proofbundle/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/b7n0de/proofbundle/blob/main/CODE_OF_CONDUCT.md). Good first
|
|
@@ -122,6 +122,7 @@ tests/test_promptfoo.py
|
|
|
122
122
|
tests/test_provenance.py
|
|
123
123
|
tests/test_pytest_plugin.py
|
|
124
124
|
tests/test_rekor_interop.py
|
|
125
|
+
tests/test_release_integrity_gate.py
|
|
125
126
|
tests/test_rfc6962_external_vectors.py
|
|
126
127
|
tests/test_schema.py
|
|
127
128
|
tests/test_sdjwt_duplicate_cnf.py
|
|
@@ -177,6 +177,14 @@ class TestEeeRecordDigest(unittest.TestCase):
|
|
|
177
177
|
rec2["model_info"]["id"] = "some-other-model"
|
|
178
178
|
rec2["evaluation_id"] = "suite/some-other-model/2026"
|
|
179
179
|
self.assertEqual(_record_digest(rec), _record_digest(rec2))
|
|
180
|
+
# M2 (6-lens review): the per-result evaluation_result_id (a provenance id that can embed the
|
|
181
|
+
# model id) must ALSO be stripped from the digest — else two records differing only in that id
|
|
182
|
+
# yield different digests = a model-id confirmation oracle.
|
|
183
|
+
self.assertNotIn("evaluation_result_id", stripped["evaluation_results"][0])
|
|
184
|
+
rec4 = self._record()
|
|
185
|
+
rec4["evaluation_results"][0]["evaluation_result_id"] = "suite/some-other-model/run-9"
|
|
186
|
+
self.assertEqual(_record_digest(rec), _record_digest(rec4),
|
|
187
|
+
"evaluation_result_id must not change the digest (M2: no id oracle)")
|
|
180
188
|
# but a tampered SCORE still changes the digest (tamper-evidence preserved)
|
|
181
189
|
rec3 = self._record()
|
|
182
190
|
rec3["evaluation_results"][0]["score_details"]["score"] = 0.91
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Tests for scripts/check_version_and_changelog.py — the release-integrity gate.
|
|
2
|
+
|
|
3
|
+
Bidirectional: a consistent release state passes; each drift class (version disagreement, missing
|
|
4
|
+
changelog section, post-tag undelivered work) fails. The post-tag-drift case uses a real throwaway git
|
|
5
|
+
repo so the M2-style "merged but never released" bug is caught by a durable test, not just live.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib.util
|
|
10
|
+
import subprocess
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
_SPEC = importlib.util.spec_from_file_location(
|
|
14
|
+
"chk", Path(__file__).resolve().parents[1] / "scripts" / "check_version_and_changelog.py")
|
|
15
|
+
chk = importlib.util.module_from_spec(_SPEC)
|
|
16
|
+
_SPEC.loader.exec_module(chk)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _write_repo(t: Path, version: str, changelog_headings: list[str]) -> None:
|
|
20
|
+
(t / "src" / "proofbundle").mkdir(parents=True, exist_ok=True)
|
|
21
|
+
(t / "pyproject.toml").write_text(f'[project]\nname = "proofbundle"\nversion = "{version}"\n', encoding="utf-8")
|
|
22
|
+
(t / "src" / "proofbundle" / "__init__.py").write_text(f'__version__ = "{version}"\n', encoding="utf-8")
|
|
23
|
+
(t / "CITATION.cff").write_text(f"cff-version: 1.2.0\nversion: {version}\n", encoding="utf-8")
|
|
24
|
+
body = "# Changelog\n\n" + "".join(f"## [{h}] - 2026-07-12\n\n- something\n\n" for h in changelog_headings)
|
|
25
|
+
(t / "CHANGELOG.md").write_text(body, encoding="utf-8")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_consistent_release_passes(tmp_path):
|
|
29
|
+
_write_repo(tmp_path, "3.0.1", ["3.0.1", "3.0.0"])
|
|
30
|
+
assert chk.check(tmp_path) == [] # no git → part 3 skipped, parts 1+2 clean
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_version_disagreement_fails(tmp_path):
|
|
34
|
+
_write_repo(tmp_path, "3.0.1", ["3.0.1"])
|
|
35
|
+
(tmp_path / "src" / "proofbundle" / "__init__.py").write_text('__version__ = "3.0.0"\n', encoding="utf-8")
|
|
36
|
+
probs = chk.check(tmp_path)
|
|
37
|
+
assert any("disagreement" in p for p in probs), probs
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_changelog_missing_current_version_fails(tmp_path):
|
|
41
|
+
_write_repo(tmp_path, "3.0.1", ["3.0.0"]) # bumped to 3.0.1 but no [3.0.1] section
|
|
42
|
+
probs = chk.check(tmp_path)
|
|
43
|
+
assert any("no `## [3.0.1]`" in p for p in probs), probs
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_post_tag_undelivered_drift_fails(tmp_path):
|
|
47
|
+
# The M2 catcher: tag v3.0.0, then a non-trivial commit, version NOT bumped, no [Unreleased] → FAIL.
|
|
48
|
+
t = tmp_path
|
|
49
|
+
|
|
50
|
+
def g(*a):
|
|
51
|
+
return subprocess.run(["git", "-C", str(t), *a], capture_output=True, text=True)
|
|
52
|
+
|
|
53
|
+
g("init", "-q", "-b", "main")
|
|
54
|
+
g("config", "user.email", "t@t")
|
|
55
|
+
g("config", "user.name", "t")
|
|
56
|
+
_write_repo(t, "3.0.0", ["3.0.0"])
|
|
57
|
+
g("add", "-A")
|
|
58
|
+
g("commit", "-qm", "release: 3.0.0")
|
|
59
|
+
g("tag", "v3.0.0")
|
|
60
|
+
(t / "src" / "proofbundle" / "adapters.py").write_text("# security fix\n", encoding="utf-8")
|
|
61
|
+
g("add", "-A")
|
|
62
|
+
g("commit", "-qm", "security(M2): strip evaluation_result_id from the EEE digest")
|
|
63
|
+
probs = chk.check(t)
|
|
64
|
+
assert any("non-trivial" in p and "no `## [Unreleased]`" in p for p in probs), probs
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_post_tag_drift_ok_when_unreleased_present(tmp_path):
|
|
68
|
+
# Same as above but with an [Unreleased] section → the drift is documented → OK.
|
|
69
|
+
t = tmp_path
|
|
70
|
+
|
|
71
|
+
def g(*a):
|
|
72
|
+
return subprocess.run(["git", "-C", str(t), *a], capture_output=True, text=True)
|
|
73
|
+
|
|
74
|
+
g("init", "-q", "-b", "main")
|
|
75
|
+
g("config", "user.email", "t@t")
|
|
76
|
+
g("config", "user.name", "t")
|
|
77
|
+
_write_repo(t, "3.0.0", ["Unreleased", "3.0.0"])
|
|
78
|
+
g("add", "-A")
|
|
79
|
+
g("commit", "-qm", "release: 3.0.0")
|
|
80
|
+
g("tag", "v3.0.0")
|
|
81
|
+
(t / "src" / "proofbundle" / "adapters.py").write_text("# security fix\n", encoding="utf-8")
|
|
82
|
+
g("add", "-A")
|
|
83
|
+
g("commit", "-qm", "security(M2): fix")
|
|
84
|
+
assert chk.check(t) == []
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
raise SystemExit(__import__("pytest").main([__file__, "-q"]))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|