proofbundle 1.3.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proofbundle-1.3.0/src/proofbundle.egg-info → proofbundle-1.7.0}/PKG-INFO +62 -24
- {proofbundle-1.3.0 → proofbundle-1.7.0}/README.md +57 -18
- {proofbundle-1.3.0 → proofbundle-1.7.0}/pyproject.toml +7 -8
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/__init__.py +16 -1
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/__init__.py +5 -1
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/eee.py +1 -1
- proofbundle-1.7.0/src/proofbundle/adapters/promptfoo.py +131 -0
- proofbundle-1.7.0/src/proofbundle/adapters/samples.py +82 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/bundle.py +20 -4
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/cli.py +114 -4
- proofbundle-1.7.0/src/proofbundle/demo.py +153 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/evalclaim.py +60 -6
- proofbundle-1.7.0/src/proofbundle/hf_evals.py +185 -0
- proofbundle-1.7.0/src/proofbundle/persample.py +270 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/sdjwt.py +5 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/statuslist.py +18 -6
- {proofbundle-1.3.0 → proofbundle-1.7.0/src/proofbundle.egg-info}/PKG-INFO +62 -24
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/SOURCES.txt +9 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/requires.txt +0 -4
- proofbundle-1.7.0/tests/test_demo.py +54 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_evalclaim.py +30 -0
- proofbundle-1.7.0/tests/test_hf_evals.py +152 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_kbjwt.py +24 -6
- proofbundle-1.7.0/tests/test_persample.py +418 -0
- proofbundle-1.7.0/tests/test_promptfoo.py +177 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_statuslist.py +36 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/LICENSE +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/setup.cfg +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/_inspect_registry.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/_integration.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/inspect_ai.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/lm_eval.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/checkpoint.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/dsse.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/eee_eval_schema.json +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/emit.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/errors.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/inspect_hook.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/intoto.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/kbjwt.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/merkle.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/py.typed +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/pytest_plugin.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/sdjwt_issue.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/signature.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/tlogproof.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/dependency_links.txt +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/entry_points.txt +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/top_level.txt +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_adapters.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_adversarial.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_bundle.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_bundle_robustness.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_checkpoint.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cli.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cli_eval.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cosignature.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cosignature_mldsa.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_eee.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_emit.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_eval_claim_schema.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_examples.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_inspect_hook.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_intoto.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_intoto_dsse.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_merkle.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_merkle_property.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_pytest_plugin.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_rekor_interop.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_rfc6962_external_vectors.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_schema.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_sdjwt_issue.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_sdjwt_reference.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_signature.py +0 -0
- {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_tlogproof.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proofbundle
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.7.0
|
|
4
4
|
Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
|
|
5
5
|
Author: Konrad Gruszka
|
|
6
6
|
License: MIT
|
|
@@ -10,18 +10,17 @@ Project-URL: Issues, https://github.com/b7n0de/proofbundle/issues
|
|
|
10
10
|
Project-URL: Changelog, https://github.com/b7n0de/proofbundle/blob/main/CHANGELOG.md
|
|
11
11
|
Project-URL: Documentation, https://github.com/b7n0de/proofbundle#readme
|
|
12
12
|
Keywords: cryptography,merkle,transparency-log,ed25519,sd-jwt,verifiable-credentials,attestation,provenance,rfc6962
|
|
13
|
-
Classifier: Development Status ::
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
23
22
|
Classifier: Topic :: Security :: Cryptography
|
|
24
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.10
|
|
25
24
|
Description-Content-Type: text/markdown
|
|
26
25
|
License-File: LICENSE
|
|
27
26
|
Requires-Dist: cryptography>=42
|
|
@@ -34,7 +33,7 @@ Requires-Dist: cryptography>=48; extra == "pq"
|
|
|
34
33
|
Provides-Extra: pytest
|
|
35
34
|
Requires-Dist: pytest>=7; extra == "pytest"
|
|
36
35
|
Provides-Extra: inspect
|
|
37
|
-
Requires-Dist: inspect_ai<0.4,>=0.3.112;
|
|
36
|
+
Requires-Dist: inspect_ai<0.4,>=0.3.112; extra == "inspect"
|
|
38
37
|
Provides-Extra: dev
|
|
39
38
|
Requires-Dist: pytest>=7; extra == "dev"
|
|
40
39
|
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
@@ -44,7 +43,7 @@ Requires-Dist: build>=1; extra == "dev"
|
|
|
44
43
|
Requires-Dist: hypothesis>=6; extra == "dev"
|
|
45
44
|
Requires-Dist: rfc8785>=0.1.4; extra == "dev"
|
|
46
45
|
Requires-Dist: sd-jwt>=0.10; extra == "dev"
|
|
47
|
-
Requires-Dist: inspect_ai<0.4,>=0.3.112;
|
|
46
|
+
Requires-Dist: inspect_ai<0.4,>=0.3.112; extra == "dev"
|
|
48
47
|
Dynamic: license-file
|
|
49
48
|
|
|
50
49
|
<div align="center">
|
|
@@ -64,13 +63,16 @@ revocation snapshots, aligned to the in-toto test-result predicate. One portable
|
|
|
64
63
|
file, no server, no network.**
|
|
65
64
|
|
|
66
65
|
[](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
|
|
66
|
+
[](LICENSE)
|
|
67
|
+
[](https://github.com/astral-sh/ruff)
|
|
68
|
+
<!-- Badges below are enabled AFTER the first PyPI release (they render broken/false before it,
|
|
69
|
+
which reads as abandonment). Un-comment on first publish — see RELEASE.md:
|
|
67
70
|
[](https://pypi.org/project/proofbundle/)
|
|
68
71
|
[](https://pypi.org/project/proofbundle/)
|
|
69
72
|
[](https://pepy.tech/project/proofbundle)
|
|
70
|
-
[](LICENSE)
|
|
71
|
-
[](https://github.com/astral-sh/ruff)
|
|
72
73
|
[](https://slsa.dev)
|
|
73
74
|
[-D6248A.svg)](https://pypi.org/project/proofbundle/)
|
|
75
|
+
-->
|
|
74
76
|
[](https://github.com/C2SP/C2SP/blob/main/tlog-cosignature.md)
|
|
75
77
|
[](scripts/mutation_check.py)
|
|
76
78
|
[](https://github.com/C2SP/C2SP/blob/main/tlog-proof.md)
|
|
@@ -82,7 +84,7 @@ file, no server, no network.**
|
|
|
82
84
|
|
|
83
85
|
**At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
|
|
84
86
|
verify` checks one self-contained `bundle.json` with three offline cryptographic
|
|
85
|
-
checks → `OK` or `FAILED`. No network, no daemon, no own crypto.
|
|
87
|
+
checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
|
|
86
88
|
|
|
87
89
|
## Contents
|
|
88
90
|
|
|
@@ -147,6 +149,8 @@ not a proof that the number is *true* or that the evaluation was well designed.
|
|
|
147
149
|
| `reproduced` | independently re-run and matched |
|
|
148
150
|
| `enclave_attested` | produced in an attested trusted execution environment |
|
|
149
151
|
|
|
152
|
+
Since v1.6 `decode_eval_claim(bundle, expected_context=...)` enforces the signed `context_binding` (cross-context replay guard), and verify-side re-checks the `samples.n == n` invariant — guarantees no longer live only in the emitter.
|
|
153
|
+
|
|
150
154
|
Full detail: **[THREAT_MODEL.md](THREAT_MODEL.md)** — what `verify` catches and what it structurally cannot.
|
|
151
155
|
|
|
152
156
|
## What it verifies
|
|
@@ -159,12 +163,19 @@ A bundle is a single JSON document. `proofbundle` checks, offline:
|
|
|
159
163
|
Certificate Transparency)
|
|
160
164
|
3. **sd-jwt** (optional) — an embedded SD-JWT selective-disclosure credential is
|
|
161
165
|
well formed, and if an issuer key is given, correctly issuer-signed
|
|
162
|
-
4. **sd-jwt-key-binding** (optional, v1.2) —
|
|
163
|
-
|
|
164
|
-
`iat`/`aud`/`nonce`/`sd_hash` are
|
|
165
|
-
disclosure set, and the signature verifies under the
|
|
166
|
-
holder key. A present-but-broken KB-JWT fails the bundle
|
|
167
|
-
|
|
166
|
+
4. **sd-jwt-key-binding** (optional, v1.2; hardened v1.3) — proof-of-possession, **fail-closed**.
|
|
167
|
+
The check runs only once the issuer signature itself verified (an unauthenticated SD-JWT can
|
|
168
|
+
never report a valid holder binding). Then: `typ` is `kb+jwt`, `iat`/`aud`/`nonce`/`sd_hash` are
|
|
169
|
+
present, `sd_hash` binds the exact presented disclosure set, and the signature verifies under the
|
|
170
|
+
issuer-bound `cnf.jwk` holder key. A present-but-broken KB-JWT fails the bundle; and if the issuer
|
|
171
|
+
**bound** a `cnf` holder key but the presentation carries **no** KB-JWT, that fails too — stripping
|
|
172
|
+
the binding is a bearer downgrade, not a valid receipt. `verify --aud/--nonce` enforce RFC 9901
|
|
173
|
+
§7.3 audience/replay binding when the relying party supplies them.
|
|
174
|
+
|
|
175
|
+
Beyond the single-file bundle, the library also verifies, all offline: a **witnessed C2SP checkpoint**
|
|
176
|
+
(`verify_witnessed_checkpoint`, Ed25519 + post-quantum ML-DSA-44 cosignatures — the quorum counts distinct
|
|
177
|
+
witness **keys**, not names, so one key under many names can never stuff a threshold), a **C2SP
|
|
178
|
+
`.tlog-proof`** (`proofbundle verify-proof`), and a **Token Status List** revocation snapshot.
|
|
168
179
|
|
|
169
180
|
The verifier treats the payload as opaque bytes. It proves that these exact
|
|
170
181
|
bytes were signed and anchored, not what they mean. That is on purpose: it keeps
|
|
@@ -203,7 +214,7 @@ flowchart LR
|
|
|
203
214
|
pip install proofbundle
|
|
204
215
|
```
|
|
205
216
|
|
|
206
|
-
Requires Python 3.
|
|
217
|
+
Requires Python 3.10+ and [`cryptography`](https://cryptography.io). Signature
|
|
207
218
|
math is delegated to `cryptography`; this project never rolls its own crypto.
|
|
208
219
|
The Merkle and SD-JWT logic is pure standard library.
|
|
209
220
|
|
|
@@ -217,7 +228,7 @@ pip install "proofbundle[sdjwt]"
|
|
|
217
228
|
## Quickstart
|
|
218
229
|
|
|
219
230
|
```bash
|
|
220
|
-
#
|
|
231
|
+
# from a git checkout (examples/ ships in the repo, not the wheel):
|
|
221
232
|
python examples/make_example.py
|
|
222
233
|
|
|
223
234
|
# verify it
|
|
@@ -228,6 +239,15 @@ proofbundle verify examples/example_bundle.json
|
|
|
228
239
|
<img src="assets/demo.svg" alt="proofbundle verify output: PASS checks and OK" width="680">
|
|
229
240
|
</div>
|
|
230
241
|
|
|
242
|
+
See the whole trust story in five seconds — pip-only, offline, in memory (honest receipt verifies, six tampers fail, a swapped sample is caught):
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
pip install proofbundle
|
|
246
|
+
proofbundle demo
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Full walkthrough incl. the forced-random-sample audit: **[docs/DEMO.md](docs/DEMO.md)**.
|
|
250
|
+
|
|
231
251
|
Machine-readable output and a non-zero exit code on failure:
|
|
232
252
|
|
|
233
253
|
```bash
|
|
@@ -321,7 +341,7 @@ Since v1.2 proofbundle also speaks the witness layer:
|
|
|
321
341
|
(Ed25519 cosignature/v1) — `verify_witnessed_checkpoint` checks a checkpoint is
|
|
322
342
|
both log-signed **and** cosigned by a quorum of distinct witnesses, offline,
|
|
323
343
|
which rules out a split view by the log operator. This is the same
|
|
324
|
-
witnessed-checkpoint pattern Rekor v2 (GA October 2025)
|
|
344
|
+
witnessed-checkpoint pattern Rekor v2 (GA October 2025) is integrating.
|
|
325
345
|
|
|
326
346
|
Since v1.3 the witness layer is complete: **ML-DSA-44 cosignatures** (C2SP type
|
|
327
347
|
0x06, FIPS 204 — post-quantum, the spec's SHOULD for new witness deployments;
|
|
@@ -480,7 +500,7 @@ threshold was met without revealing the model or the data. See [INTEROP.md](INTE
|
|
|
480
500
|
[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) `results_*.json` (the
|
|
481
501
|
genuine `acc,none` filter-suffix format). **`proofbundle.adapters.from_eee_dataset`** (v0.9) reads an
|
|
482
502
|
Every Eval Ever v0.2.2 aggregate JSON and builds a signed receipt — validated against the vendored EEE
|
|
483
|
-
schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.
|
|
503
|
+
schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.10+).
|
|
484
504
|
- **in-toto test-result export, DSSE-signed** (v0.9) — `proofbundle.intoto.export_intoto_dsse(claim,
|
|
485
505
|
signer)` emits the receipt as a DSSE-signed in-toto Statement v1 with the **generic
|
|
486
506
|
`test-result/v0.1` predicate** (result PASSED/FAILED, `configuration` ResourceDescriptors), so a generic
|
|
@@ -540,15 +560,33 @@ ISO/IEC DIS 24970), including the anti-patterns no one should claim.
|
|
|
540
560
|
(RFC 9901 §4.3, fail-closed; closes #1) with `cnf.jwk` issuance and holder presentation, **C2SP
|
|
541
561
|
tlog-cosignature** verification (Ed25519 cosignature/v1, witness quorum, split-view resistance), and
|
|
542
562
|
`verify --verbose` with the recomputed Merkle root (closes #2).
|
|
543
|
-
- **v1.3
|
|
563
|
+
- **v1.3** — the portable proof: **C2SP tlog-proof** emit/verify (+
|
|
544
564
|
`verify-proof` CLI), **ML-DSA-44** witness cosignatures (post-quantum, `[pq]` extra), **Token
|
|
545
565
|
Status List** snapshot verification (offline revocation), SD-JWT VC markers (`dc+sd-jwt`,
|
|
546
566
|
`vct`, `status`), COMPLIANCE.md (EU AI Act Art. 12 / NIST AI RMF mapping), and the mutation
|
|
547
567
|
suite as a CI gate.
|
|
548
|
-
- **
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
568
|
+
- **v1.4** — distribution: a **promptfoo** adapter (results.json v3 →
|
|
569
|
+
pass_rate receipt), the **Hugging Face Community Evals bridge** (`pb1.` receipt tokens for the
|
|
570
|
+
`.eval_results` `verifyToken` field + a strict YAML entry emitter — proofbundle-verifiable,
|
|
571
|
+
explicitly NOT the HF-internal badge token), `hf-token` CLI, and the **Python 3.10+ floor**
|
|
572
|
+
(3.9 is EOL since 2025-10).
|
|
573
|
+
- **v1.5** — per-sample receipts: a signed **Merkle commitment over every
|
|
574
|
+
individual sample** (`samples` claim field, SPEC §7g), salted per-leaf disclosures (RFC 9901
|
|
575
|
+
mechanic), **sample openings** with replay guards, and a **spot-check audit protocol**
|
|
576
|
+
(`audit-challenge` / `verify-opening` CLI; auditor-nonce, beacon, and documented-grindable
|
|
577
|
+
self-challenge modes; PoR soundness 1−(1−m)^k) — narrowing the THREAT_MODEL's sub-sampling gap.
|
|
578
|
+
Extractors for lm-eval `--log_samples` JSONL and promptfoo rows.
|
|
579
|
+
- **v1.6** — external principal-security review fixes: verify-side sample/`leaf_alg`/root invariants
|
|
580
|
+
and signed `context_binding` enforcement (was signed but unchecked), status-list freshness
|
|
581
|
+
(`exp`/`ttl` None-safe), `merkle.hash_alg` now required; Beta classifier; 26-operator mutation gate.
|
|
582
|
+
- **v1.6.1** — developer experience: `proofbundle demo` (pip-only, fail-closed tamper smoke test), a
|
|
583
|
+
runnable per-sample example, `docs/DEMO.md`.
|
|
584
|
+
- **v1.7 (current release)** — release supply-chain hardening: `release.yml` builds once and
|
|
585
|
+
publishes the **attested** bytes (sha256 gate: attested == published), a reviewer-gated `pypi`
|
|
586
|
+
environment, least-privilege permissions, SHA-pinned actions, `RELEASE.md` + `docs/REVIEWERS.md`.
|
|
587
|
+
- **Deferred** (explicitly not yet built, stated honestly) — full SD-JWT VC conformance + `vct`
|
|
588
|
+
type-metadata resolution (pre-IESG), an official in-toto eval predicate (proposal path via
|
|
589
|
+
OpenSSF/CoSAI), and a full in-toto client.
|
|
552
590
|
|
|
553
591
|
## Contributing
|
|
554
592
|
|
|
@@ -15,13 +15,16 @@ revocation snapshots, aligned to the in-toto test-result predicate. One portable
|
|
|
15
15
|
file, no server, no network.**
|
|
16
16
|
|
|
17
17
|
[](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
|
|
18
|
+
[](LICENSE)
|
|
19
|
+
[](https://github.com/astral-sh/ruff)
|
|
20
|
+
<!-- Badges below are enabled AFTER the first PyPI release (they render broken/false before it,
|
|
21
|
+
which reads as abandonment). Un-comment on first publish — see RELEASE.md:
|
|
18
22
|
[](https://pypi.org/project/proofbundle/)
|
|
19
23
|
[](https://pypi.org/project/proofbundle/)
|
|
20
24
|
[](https://pepy.tech/project/proofbundle)
|
|
21
|
-
[](LICENSE)
|
|
22
|
-
[](https://github.com/astral-sh/ruff)
|
|
23
25
|
[](https://slsa.dev)
|
|
24
26
|
[-D6248A.svg)](https://pypi.org/project/proofbundle/)
|
|
27
|
+
-->
|
|
25
28
|
[](https://github.com/C2SP/C2SP/blob/main/tlog-cosignature.md)
|
|
26
29
|
[](scripts/mutation_check.py)
|
|
27
30
|
[](https://github.com/C2SP/C2SP/blob/main/tlog-proof.md)
|
|
@@ -33,7 +36,7 @@ file, no server, no network.**
|
|
|
33
36
|
|
|
34
37
|
**At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
|
|
35
38
|
verify` checks one self-contained `bundle.json` with three offline cryptographic
|
|
36
|
-
checks → `OK` or `FAILED`. No network, no daemon, no own crypto.
|
|
39
|
+
checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
|
|
37
40
|
|
|
38
41
|
## Contents
|
|
39
42
|
|
|
@@ -98,6 +101,8 @@ not a proof that the number is *true* or that the evaluation was well designed.
|
|
|
98
101
|
| `reproduced` | independently re-run and matched |
|
|
99
102
|
| `enclave_attested` | produced in an attested trusted execution environment |
|
|
100
103
|
|
|
104
|
+
Since v1.6 `decode_eval_claim(bundle, expected_context=...)` enforces the signed `context_binding` (cross-context replay guard), and verify-side re-checks the `samples.n == n` invariant — guarantees no longer live only in the emitter.
|
|
105
|
+
|
|
101
106
|
Full detail: **[THREAT_MODEL.md](THREAT_MODEL.md)** — what `verify` catches and what it structurally cannot.
|
|
102
107
|
|
|
103
108
|
## What it verifies
|
|
@@ -110,12 +115,19 @@ A bundle is a single JSON document. `proofbundle` checks, offline:
|
|
|
110
115
|
Certificate Transparency)
|
|
111
116
|
3. **sd-jwt** (optional) — an embedded SD-JWT selective-disclosure credential is
|
|
112
117
|
well formed, and if an issuer key is given, correctly issuer-signed
|
|
113
|
-
4. **sd-jwt-key-binding** (optional, v1.2) —
|
|
114
|
-
|
|
115
|
-
`iat`/`aud`/`nonce`/`sd_hash` are
|
|
116
|
-
disclosure set, and the signature verifies under the
|
|
117
|
-
holder key. A present-but-broken KB-JWT fails the bundle
|
|
118
|
-
|
|
118
|
+
4. **sd-jwt-key-binding** (optional, v1.2; hardened v1.3) — proof-of-possession, **fail-closed**.
|
|
119
|
+
The check runs only once the issuer signature itself verified (an unauthenticated SD-JWT can
|
|
120
|
+
never report a valid holder binding). Then: `typ` is `kb+jwt`, `iat`/`aud`/`nonce`/`sd_hash` are
|
|
121
|
+
present, `sd_hash` binds the exact presented disclosure set, and the signature verifies under the
|
|
122
|
+
issuer-bound `cnf.jwk` holder key. A present-but-broken KB-JWT fails the bundle; and if the issuer
|
|
123
|
+
**bound** a `cnf` holder key but the presentation carries **no** KB-JWT, that fails too — stripping
|
|
124
|
+
the binding is a bearer downgrade, not a valid receipt. `verify --aud/--nonce` enforce RFC 9901
|
|
125
|
+
§7.3 audience/replay binding when the relying party supplies them.
|
|
126
|
+
|
|
127
|
+
Beyond the single-file bundle, the library also verifies, all offline: a **witnessed C2SP checkpoint**
|
|
128
|
+
(`verify_witnessed_checkpoint`, Ed25519 + post-quantum ML-DSA-44 cosignatures — the quorum counts distinct
|
|
129
|
+
witness **keys**, not names, so one key under many names can never stuff a threshold), a **C2SP
|
|
130
|
+
`.tlog-proof`** (`proofbundle verify-proof`), and a **Token Status List** revocation snapshot.
|
|
119
131
|
|
|
120
132
|
The verifier treats the payload as opaque bytes. It proves that these exact
|
|
121
133
|
bytes were signed and anchored, not what they mean. That is on purpose: it keeps
|
|
@@ -154,7 +166,7 @@ flowchart LR
|
|
|
154
166
|
pip install proofbundle
|
|
155
167
|
```
|
|
156
168
|
|
|
157
|
-
Requires Python 3.
|
|
169
|
+
Requires Python 3.10+ and [`cryptography`](https://cryptography.io). Signature
|
|
158
170
|
math is delegated to `cryptography`; this project never rolls its own crypto.
|
|
159
171
|
The Merkle and SD-JWT logic is pure standard library.
|
|
160
172
|
|
|
@@ -168,7 +180,7 @@ pip install "proofbundle[sdjwt]"
|
|
|
168
180
|
## Quickstart
|
|
169
181
|
|
|
170
182
|
```bash
|
|
171
|
-
#
|
|
183
|
+
# from a git checkout (examples/ ships in the repo, not the wheel):
|
|
172
184
|
python examples/make_example.py
|
|
173
185
|
|
|
174
186
|
# verify it
|
|
@@ -179,6 +191,15 @@ proofbundle verify examples/example_bundle.json
|
|
|
179
191
|
<img src="assets/demo.svg" alt="proofbundle verify output: PASS checks and OK" width="680">
|
|
180
192
|
</div>
|
|
181
193
|
|
|
194
|
+
See the whole trust story in five seconds — pip-only, offline, in memory (honest receipt verifies, six tampers fail, a swapped sample is caught):
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install proofbundle
|
|
198
|
+
proofbundle demo
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Full walkthrough incl. the forced-random-sample audit: **[docs/DEMO.md](docs/DEMO.md)**.
|
|
202
|
+
|
|
182
203
|
Machine-readable output and a non-zero exit code on failure:
|
|
183
204
|
|
|
184
205
|
```bash
|
|
@@ -272,7 +293,7 @@ Since v1.2 proofbundle also speaks the witness layer:
|
|
|
272
293
|
(Ed25519 cosignature/v1) — `verify_witnessed_checkpoint` checks a checkpoint is
|
|
273
294
|
both log-signed **and** cosigned by a quorum of distinct witnesses, offline,
|
|
274
295
|
which rules out a split view by the log operator. This is the same
|
|
275
|
-
witnessed-checkpoint pattern Rekor v2 (GA October 2025)
|
|
296
|
+
witnessed-checkpoint pattern Rekor v2 (GA October 2025) is integrating.
|
|
276
297
|
|
|
277
298
|
Since v1.3 the witness layer is complete: **ML-DSA-44 cosignatures** (C2SP type
|
|
278
299
|
0x06, FIPS 204 — post-quantum, the spec's SHOULD for new witness deployments;
|
|
@@ -431,7 +452,7 @@ threshold was met without revealing the model or the data. See [INTEROP.md](INTE
|
|
|
431
452
|
[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) `results_*.json` (the
|
|
432
453
|
genuine `acc,none` filter-suffix format). **`proofbundle.adapters.from_eee_dataset`** (v0.9) reads an
|
|
433
454
|
Every Eval Ever v0.2.2 aggregate JSON and builds a signed receipt — validated against the vendored EEE
|
|
434
|
-
schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.
|
|
455
|
+
schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.10+).
|
|
435
456
|
- **in-toto test-result export, DSSE-signed** (v0.9) — `proofbundle.intoto.export_intoto_dsse(claim,
|
|
436
457
|
signer)` emits the receipt as a DSSE-signed in-toto Statement v1 with the **generic
|
|
437
458
|
`test-result/v0.1` predicate** (result PASSED/FAILED, `configuration` ResourceDescriptors), so a generic
|
|
@@ -491,15 +512,33 @@ ISO/IEC DIS 24970), including the anti-patterns no one should claim.
|
|
|
491
512
|
(RFC 9901 §4.3, fail-closed; closes #1) with `cnf.jwk` issuance and holder presentation, **C2SP
|
|
492
513
|
tlog-cosignature** verification (Ed25519 cosignature/v1, witness quorum, split-view resistance), and
|
|
493
514
|
`verify --verbose` with the recomputed Merkle root (closes #2).
|
|
494
|
-
- **v1.3
|
|
515
|
+
- **v1.3** — the portable proof: **C2SP tlog-proof** emit/verify (+
|
|
495
516
|
`verify-proof` CLI), **ML-DSA-44** witness cosignatures (post-quantum, `[pq]` extra), **Token
|
|
496
517
|
Status List** snapshot verification (offline revocation), SD-JWT VC markers (`dc+sd-jwt`,
|
|
497
518
|
`vct`, `status`), COMPLIANCE.md (EU AI Act Art. 12 / NIST AI RMF mapping), and the mutation
|
|
498
519
|
suite as a CI gate.
|
|
499
|
-
- **
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
520
|
+
- **v1.4** — distribution: a **promptfoo** adapter (results.json v3 →
|
|
521
|
+
pass_rate receipt), the **Hugging Face Community Evals bridge** (`pb1.` receipt tokens for the
|
|
522
|
+
`.eval_results` `verifyToken` field + a strict YAML entry emitter — proofbundle-verifiable,
|
|
523
|
+
explicitly NOT the HF-internal badge token), `hf-token` CLI, and the **Python 3.10+ floor**
|
|
524
|
+
(3.9 is EOL since 2025-10).
|
|
525
|
+
- **v1.5** — per-sample receipts: a signed **Merkle commitment over every
|
|
526
|
+
individual sample** (`samples` claim field, SPEC §7g), salted per-leaf disclosures (RFC 9901
|
|
527
|
+
mechanic), **sample openings** with replay guards, and a **spot-check audit protocol**
|
|
528
|
+
(`audit-challenge` / `verify-opening` CLI; auditor-nonce, beacon, and documented-grindable
|
|
529
|
+
self-challenge modes; PoR soundness 1−(1−m)^k) — narrowing the THREAT_MODEL's sub-sampling gap.
|
|
530
|
+
Extractors for lm-eval `--log_samples` JSONL and promptfoo rows.
|
|
531
|
+
- **v1.6** — external principal-security review fixes: verify-side sample/`leaf_alg`/root invariants
|
|
532
|
+
and signed `context_binding` enforcement (was signed but unchecked), status-list freshness
|
|
533
|
+
(`exp`/`ttl` None-safe), `merkle.hash_alg` now required; Beta classifier; 26-operator mutation gate.
|
|
534
|
+
- **v1.6.1** — developer experience: `proofbundle demo` (pip-only, fail-closed tamper smoke test), a
|
|
535
|
+
runnable per-sample example, `docs/DEMO.md`.
|
|
536
|
+
- **v1.7 (current release)** — release supply-chain hardening: `release.yml` builds once and
|
|
537
|
+
publishes the **attested** bytes (sha256 gate: attested == published), a reviewer-gated `pypi`
|
|
538
|
+
environment, least-privilege permissions, SHA-pinned actions, `RELEASE.md` + `docs/REVIEWERS.md`.
|
|
539
|
+
- **Deferred** (explicitly not yet built, stated honestly) — full SD-JWT VC conformance + `vct`
|
|
540
|
+
type-metadata resolution (pre-IESG), an official in-toto eval predicate (proposal path via
|
|
541
|
+
OpenSSF/CoSAI), and a full in-toto client.
|
|
503
542
|
|
|
504
543
|
## Contributing
|
|
505
544
|
|
|
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "proofbundle"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.7.0"
|
|
8
8
|
description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = { text = "MIT" }
|
|
12
12
|
authors = [{ name = "Konrad Gruszka" }]
|
|
13
13
|
keywords = [
|
|
@@ -22,11 +22,10 @@ keywords = [
|
|
|
22
22
|
"rfc6962",
|
|
23
23
|
]
|
|
24
24
|
classifiers = [
|
|
25
|
-
"Development Status ::
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
26
|
"Intended Audience :: Developers",
|
|
27
27
|
"License :: OSI Approved :: MIT License",
|
|
28
28
|
"Programming Language :: Python :: 3",
|
|
29
|
-
"Programming Language :: Python :: 3.9",
|
|
30
29
|
"Programming Language :: Python :: 3.10",
|
|
31
30
|
"Programming Language :: Python :: 3.11",
|
|
32
31
|
"Programming Language :: Python :: 3.12",
|
|
@@ -56,11 +55,11 @@ pq = ["cryptography>=48"]
|
|
|
56
55
|
pytest = ["pytest>=7"]
|
|
57
56
|
# The inspect_ai adapter uses the STABLE read_eval_log API (lazy import). Pinned with an UPPER bound:
|
|
58
57
|
# the .eval format + pydantic schema change between versions (inspect_ai issue 834), and the fixture
|
|
59
|
-
# test is bound to this range.
|
|
60
|
-
#
|
|
61
|
-
inspect = ['inspect_ai>=0.3.112,<0.4
|
|
58
|
+
# test is bound to this range. Since v1.4.0 the project floor is Python 3.10, so no per-extra version
|
|
59
|
+
# marker is needed here (inspect_ai's own >=3.10 requirement is always satisfiable on a supported base).
|
|
60
|
+
inspect = ['inspect_ai>=0.3.112,<0.4']
|
|
62
61
|
dev = ["pytest>=7", "ruff>=0.5", "jsonschema>=4", "mypy>=1.8", "build>=1", "hypothesis>=6",
|
|
63
|
-
"rfc8785>=0.1.4", "sd-jwt>=0.10", 'inspect_ai>=0.3.112,<0.4
|
|
62
|
+
"rfc8785>=0.1.4", "sd-jwt>=0.10", 'inspect_ai>=0.3.112,<0.4']
|
|
64
63
|
|
|
65
64
|
[project.urls]
|
|
66
65
|
Homepage = "https://b7n0de.com"
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from typing import TYPE_CHECKING
|
|
15
15
|
|
|
16
|
-
__version__ = "1.
|
|
16
|
+
__version__ = "1.7.0"
|
|
17
17
|
|
|
18
18
|
__all__ = [
|
|
19
19
|
"__version__",
|
|
@@ -30,6 +30,12 @@ __all__ = [
|
|
|
30
30
|
"verify_cosignature",
|
|
31
31
|
"verify_witnessed_checkpoint",
|
|
32
32
|
"verify_status_snapshot",
|
|
33
|
+
"receipt_token",
|
|
34
|
+
"verify_receipt_token",
|
|
35
|
+
"build_sample_tree",
|
|
36
|
+
"sample_opening",
|
|
37
|
+
"verify_sample_opening",
|
|
38
|
+
"audit_challenge",
|
|
33
39
|
"VerificationResult",
|
|
34
40
|
"Check",
|
|
35
41
|
"ProofBundleError",
|
|
@@ -47,6 +53,12 @@ _LAZY = {
|
|
|
47
53
|
"verify_cosignature": ".checkpoint",
|
|
48
54
|
"verify_witnessed_checkpoint": ".checkpoint",
|
|
49
55
|
"verify_status_snapshot": ".statuslist",
|
|
56
|
+
"receipt_token": ".hf_evals",
|
|
57
|
+
"verify_receipt_token": ".hf_evals",
|
|
58
|
+
"build_sample_tree": ".persample",
|
|
59
|
+
"sample_opening": ".persample",
|
|
60
|
+
"verify_sample_opening": ".persample",
|
|
61
|
+
"audit_challenge": ".persample",
|
|
50
62
|
}
|
|
51
63
|
|
|
52
64
|
if TYPE_CHECKING: # static analysers + IDEs see the real names/types; runtime stays lazy
|
|
@@ -55,6 +67,9 @@ if TYPE_CHECKING: # static analysers + IDEs see the real names/types; runtime s
|
|
|
55
67
|
from .errors import Check, ProofBundleError, VerificationResult
|
|
56
68
|
from .checkpoint import verify_cosignature, verify_witnessed_checkpoint
|
|
57
69
|
from .kbjwt import verify_key_binding
|
|
70
|
+
from .hf_evals import receipt_token, verify_receipt_token
|
|
71
|
+
from .persample import (audit_challenge, build_sample_tree, sample_opening,
|
|
72
|
+
verify_sample_opening)
|
|
58
73
|
from .statuslist import verify_status_snapshot
|
|
59
74
|
from .tlogproof import verify_tlog_proof
|
|
60
75
|
from .merkle import verify_consistency, verify_inclusion
|
|
@@ -7,5 +7,9 @@ each fixture in tests/fixtures documents its source + version.
|
|
|
7
7
|
from .inspect_ai import from_inspect_ai_log
|
|
8
8
|
from .eee import from_eee_dataset
|
|
9
9
|
from .lm_eval import from_lm_eval_results
|
|
10
|
+
from .promptfoo import from_promptfoo_results
|
|
11
|
+
from .samples import samples_from_lm_eval_jsonl, samples_from_promptfoo_results
|
|
10
12
|
|
|
11
|
-
__all__ = ["from_lm_eval_results", "from_inspect_ai_log", "from_eee_dataset"
|
|
13
|
+
__all__ = ["from_lm_eval_results", "from_inspect_ai_log", "from_eee_dataset",
|
|
14
|
+
"from_promptfoo_results",
|
|
15
|
+
"samples_from_lm_eval_jsonl", "samples_from_promptfoo_results"]
|
|
@@ -5,7 +5,7 @@ it has no cryptography. This converter is strictly additive: it reads an EEE agg
|
|
|
5
5
|
signed, selectively-disclosable proofbundle receipt from it.
|
|
6
6
|
|
|
7
7
|
IMPORTANT: `every_eval_ever` is NOT imported at runtime — it requires Python 3.12+ (pydantic/numpy/pandas/
|
|
8
|
-
duckdb), while proofbundle stays 3.
|
|
8
|
+
duckdb), while proofbundle stays 3.10+. We parse the EEE JSON directly and OPTIONALLY validate it against the
|
|
9
9
|
vendored `eee_eval_schema.json` (schema version 0.2.2, MIT) using `jsonschema` if available.
|
|
10
10
|
|
|
11
11
|
Field mapping (verified 2026-07 against schemas/eval.schema.json v0.2.2):
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Adapter for promptfoo `eval -o results.json` output (file-based, NO promptfoo import) — v1.4.
|
|
2
|
+
|
|
3
|
+
Parses the exported OutputFile JSON only. Format verified 2026-07-02 against promptfoo `main`
|
|
4
|
+
(`src/types/index.ts` OutputFile / EvaluateSummaryV3 / EvaluateStats, `src/util/output.ts`
|
|
5
|
+
writeOutput): top level carries `evalId`, `results`, `config`, `metadata`; the current summary is
|
|
6
|
+
**version 3** (`results.version == 3`) with whole-run aggregates in `results.stats` as
|
|
7
|
+
`successes` / `failures` / `errors`. The metric emitted is **pass_rate** = successes / (successes
|
|
8
|
+
+ failures + errors), rendered as a fixed-point decimal string (the claim schema takes decimal
|
|
9
|
+
strings, never floats).
|
|
10
|
+
|
|
11
|
+
Version honesty: legacy files with `results.version` 1/2 have a different shape (`table` instead
|
|
12
|
+
of `prompts`) and are REJECTED with a clear message rather than half-parsed; the "v4" storage
|
|
13
|
+
version never appears in output files (they still say `version: 3`). Redaction flags
|
|
14
|
+
(`PROMPTFOO_STRIP_*`) only affect prompt/response bodies — nothing this adapter reads.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
from decimal import Decimal
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Optional
|
|
23
|
+
|
|
24
|
+
from ..evalclaim import build_eval_claim
|
|
25
|
+
|
|
26
|
+
_SCALE = 6 # pass_rate decimal places — fixed-point, schema-conformant
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _pass_rate(successes: int, failures: int, errors: int) -> "tuple[str, int]":
|
|
30
|
+
total = successes + failures + errors
|
|
31
|
+
if total <= 0:
|
|
32
|
+
raise ValueError("promptfoo results contain no test outcomes (stats all zero)")
|
|
33
|
+
rate = (Decimal(successes) / Decimal(total)).quantize(Decimal(1).scaleb(-_SCALE))
|
|
34
|
+
return f"{rate:.{_SCALE}f}", total
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def from_promptfoo_results(path, *, comparator: str, threshold: str, timestamp: str,
|
|
38
|
+
model_salt: Optional[bytes] = None,
|
|
39
|
+
dataset_salt: Optional[bytes] = None):
|
|
40
|
+
"""Read a promptfoo results.json (summary v3) and build an eval claim over the run's
|
|
41
|
+
pass_rate. Returns (claim, salts).
|
|
42
|
+
|
|
43
|
+
- suite = `config.description` when present, else the `evalId` (the run's identity).
|
|
44
|
+
- model_id = the sorted, de-duplicated provider ids of the run (a promptfoo eval may span
|
|
45
|
+
several providers; the salted commitment pins the exact set).
|
|
46
|
+
- dataset_id = sha256 over the canonical JSON of `config.tests` AS RECORDED in results.json.
|
|
47
|
+
Inline tests bind the actual content (the suite IS the dataset); a `file://`/glob reference binds
|
|
48
|
+
the reference, not the resolved file content (an offline reader cannot resolve it) — provenance
|
|
49
|
+
records which case applies (`dataset_commitment_scope`) so the binding is never overstated.
|
|
50
|
+
- provenance: promptfooVersion, evalId, summary timestamp, per-outcome counts.
|
|
51
|
+
"""
|
|
52
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
53
|
+
if not isinstance(data, dict) or not isinstance(data.get("results"), dict):
|
|
54
|
+
raise ValueError("not a promptfoo output file (missing results object)")
|
|
55
|
+
summary = data["results"]
|
|
56
|
+
version = summary.get("version")
|
|
57
|
+
if version != 3:
|
|
58
|
+
raise ValueError(
|
|
59
|
+
f"unsupported promptfoo summary version {version!r} — this adapter parses the "
|
|
60
|
+
"current version 3 (legacy v1/v2 files use a different 'table' shape; re-export "
|
|
61
|
+
"with a current promptfoo)")
|
|
62
|
+
stats = summary.get("stats")
|
|
63
|
+
if not isinstance(stats, dict):
|
|
64
|
+
raise ValueError("promptfoo results.stats missing")
|
|
65
|
+
counts = {}
|
|
66
|
+
for key in ("successes", "failures", "errors"):
|
|
67
|
+
val = stats.get(key, 0)
|
|
68
|
+
if isinstance(val, bool) or not isinstance(val, int) or val < 0:
|
|
69
|
+
raise ValueError(f"promptfoo stats.{key} must be a non-negative integer")
|
|
70
|
+
counts[key] = val
|
|
71
|
+
score, n = _pass_rate(counts["successes"], counts["failures"], counts["errors"])
|
|
72
|
+
|
|
73
|
+
config = data.get("config") or {}
|
|
74
|
+
eval_id = data.get("evalId")
|
|
75
|
+
suite = str(config.get("description") or eval_id or "promptfoo-run")
|
|
76
|
+
|
|
77
|
+
# Commit to the providers that ACTUALLY produced a result in this run — not config.providers, which can
|
|
78
|
+
# list providers that never ran (e.g. --filter-providers). Only fall back to config.providers if the
|
|
79
|
+
# summary recorded no per-result provider at all (release-review fix).
|
|
80
|
+
providers = set()
|
|
81
|
+
for res in summary.get("results", []):
|
|
82
|
+
prov = (res or {}).get("provider")
|
|
83
|
+
if isinstance(prov, dict) and prov.get("id"):
|
|
84
|
+
providers.add(str(prov["id"]))
|
|
85
|
+
elif isinstance(prov, str):
|
|
86
|
+
providers.add(prov)
|
|
87
|
+
if not providers:
|
|
88
|
+
for prov in config.get("providers") or []:
|
|
89
|
+
if isinstance(prov, str):
|
|
90
|
+
providers.add(prov)
|
|
91
|
+
elif isinstance(prov, dict) and prov.get("id"):
|
|
92
|
+
providers.add(str(prov["id"]))
|
|
93
|
+
model_id = "+".join(sorted(providers)) if providers else "unknown:promptfoo-provider"
|
|
94
|
+
|
|
95
|
+
tests = config.get("tests")
|
|
96
|
+
tests_canonical = json.dumps(tests, sort_keys=True, separators=(",", ":")) if tests else ""
|
|
97
|
+
dataset_id = ("promptfoo-tests-sha256:"
|
|
98
|
+
+ hashlib.sha256(tests_canonical.encode("utf-8")).hexdigest())
|
|
99
|
+
# HONESTY (release-review): the commitment is over `config.tests` AS RECORDED in results.json. When tests are
|
|
100
|
+
# INLINE, that is the content (the suite IS the dataset). When they are a `file://`/glob REFERENCE — whether a
|
|
101
|
+
# top-level string, a list containing a string, OR a `file://` nested inside an inline test's vars/assert —
|
|
102
|
+
# promptfoo records the unresolved reference, so the commitment binds the reference, NOT the file content
|
|
103
|
+
# (an offline reader cannot resolve external files). When there are NO tests at all the commitment is over the
|
|
104
|
+
# empty string and binds nothing. We label each case so a verifier is never misled that content was bound.
|
|
105
|
+
tests_by_reference = isinstance(tests, str) or (
|
|
106
|
+
isinstance(tests, list) and any(isinstance(t, str) for t in tests)) or ("file://" in tests_canonical)
|
|
107
|
+
if not tests:
|
|
108
|
+
commitment_scope = "config.tests_absent"
|
|
109
|
+
elif tests_by_reference:
|
|
110
|
+
commitment_scope = "config.tests_reference_only"
|
|
111
|
+
else:
|
|
112
|
+
commitment_scope = "config.tests_inline_content"
|
|
113
|
+
|
|
114
|
+
metadata = data.get("metadata") or {}
|
|
115
|
+
provenance = {"harness": "promptfoo",
|
|
116
|
+
"successes": str(counts["successes"]), "failures": str(counts["failures"]),
|
|
117
|
+
"errors": str(counts["errors"]),
|
|
118
|
+
"pass_rate_formula": "successes/(successes+failures+errors)",
|
|
119
|
+
"dataset_commitment_scope": commitment_scope}
|
|
120
|
+
if eval_id:
|
|
121
|
+
provenance["eval_id"] = str(eval_id)
|
|
122
|
+
if metadata.get("promptfooVersion"):
|
|
123
|
+
provenance["promptfoo_version"] = str(metadata["promptfooVersion"])
|
|
124
|
+
if summary.get("timestamp"):
|
|
125
|
+
provenance["run_timestamp"] = str(summary["timestamp"])
|
|
126
|
+
|
|
127
|
+
return build_eval_claim(
|
|
128
|
+
suite=suite, suite_version=f"promptfoo-summary-v{version}",
|
|
129
|
+
metric="pass_rate", comparator=comparator, threshold=threshold, score=score, n=n,
|
|
130
|
+
model_id=model_id, dataset_id=dataset_id, issuer="", timestamp=timestamp,
|
|
131
|
+
provenance=provenance, model_salt=model_salt, dataset_salt=dataset_salt)
|