proofbundle 1.3.0__tar.gz → 1.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {proofbundle-1.3.0/src/proofbundle.egg-info → proofbundle-1.7.0}/PKG-INFO +62 -24
  2. {proofbundle-1.3.0 → proofbundle-1.7.0}/README.md +57 -18
  3. {proofbundle-1.3.0 → proofbundle-1.7.0}/pyproject.toml +7 -8
  4. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/__init__.py +16 -1
  5. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/__init__.py +5 -1
  6. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/eee.py +1 -1
  7. proofbundle-1.7.0/src/proofbundle/adapters/promptfoo.py +131 -0
  8. proofbundle-1.7.0/src/proofbundle/adapters/samples.py +82 -0
  9. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/bundle.py +20 -4
  10. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/cli.py +114 -4
  11. proofbundle-1.7.0/src/proofbundle/demo.py +153 -0
  12. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/evalclaim.py +60 -6
  13. proofbundle-1.7.0/src/proofbundle/hf_evals.py +185 -0
  14. proofbundle-1.7.0/src/proofbundle/persample.py +270 -0
  15. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/sdjwt.py +5 -0
  16. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/statuslist.py +18 -6
  17. {proofbundle-1.3.0 → proofbundle-1.7.0/src/proofbundle.egg-info}/PKG-INFO +62 -24
  18. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/SOURCES.txt +9 -0
  19. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/requires.txt +0 -4
  20. proofbundle-1.7.0/tests/test_demo.py +54 -0
  21. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_evalclaim.py +30 -0
  22. proofbundle-1.7.0/tests/test_hf_evals.py +152 -0
  23. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_kbjwt.py +24 -6
  24. proofbundle-1.7.0/tests/test_persample.py +418 -0
  25. proofbundle-1.7.0/tests/test_promptfoo.py +177 -0
  26. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_statuslist.py +36 -0
  27. {proofbundle-1.3.0 → proofbundle-1.7.0}/LICENSE +0 -0
  28. {proofbundle-1.3.0 → proofbundle-1.7.0}/setup.cfg +0 -0
  29. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/_inspect_registry.py +0 -0
  30. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/_integration.py +0 -0
  31. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/inspect_ai.py +0 -0
  32. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/adapters/lm_eval.py +0 -0
  33. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/checkpoint.py +0 -0
  34. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/dsse.py +0 -0
  35. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/eee_eval_schema.json +0 -0
  36. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/emit.py +0 -0
  37. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/errors.py +0 -0
  38. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/inspect_hook.py +0 -0
  39. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/intoto.py +0 -0
  40. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/kbjwt.py +0 -0
  41. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/merkle.py +0 -0
  42. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/py.typed +0 -0
  43. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/pytest_plugin.py +0 -0
  44. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/sdjwt_issue.py +0 -0
  45. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/signature.py +0 -0
  46. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle/tlogproof.py +0 -0
  47. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/dependency_links.txt +0 -0
  48. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/entry_points.txt +0 -0
  49. {proofbundle-1.3.0 → proofbundle-1.7.0}/src/proofbundle.egg-info/top_level.txt +0 -0
  50. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_adapters.py +0 -0
  51. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_adversarial.py +0 -0
  52. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_bundle.py +0 -0
  53. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_bundle_robustness.py +0 -0
  54. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_checkpoint.py +0 -0
  55. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cli.py +0 -0
  56. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cli_eval.py +0 -0
  57. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cosignature.py +0 -0
  58. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_cosignature_mldsa.py +0 -0
  59. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_eee.py +0 -0
  60. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_emit.py +0 -0
  61. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_eval_claim_schema.py +0 -0
  62. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_examples.py +0 -0
  63. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_inspect_hook.py +0 -0
  64. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_intoto.py +0 -0
  65. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_intoto_dsse.py +0 -0
  66. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_merkle.py +0 -0
  67. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_merkle_property.py +0 -0
  68. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_pytest_plugin.py +0 -0
  69. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_rekor_interop.py +0 -0
  70. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_rfc6962_external_vectors.py +0 -0
  71. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_schema.py +0 -0
  72. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_sdjwt_issue.py +0 -0
  73. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_sdjwt_reference.py +0 -0
  74. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_signature.py +0 -0
  75. {proofbundle-1.3.0 → proofbundle-1.7.0}/tests/test_tlogproof.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: proofbundle
3
- Version: 1.3.0
3
+ Version: 1.7.0
4
4
  Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
5
5
  Author: Konrad Gruszka
6
6
  License: MIT
@@ -10,18 +10,17 @@ Project-URL: Issues, https://github.com/b7n0de/proofbundle/issues
10
10
  Project-URL: Changelog, https://github.com/b7n0de/proofbundle/blob/main/CHANGELOG.md
11
11
  Project-URL: Documentation, https://github.com/b7n0de/proofbundle#readme
12
12
  Keywords: cryptography,merkle,transparency-log,ed25519,sd-jwt,verifiable-credentials,attestation,provenance,rfc6962
13
- Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Development Status :: 4 - Beta
14
14
  Classifier: Intended Audience :: Developers
15
15
  Classifier: License :: OSI Approved :: MIT License
16
16
  Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3.9
18
17
  Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
22
21
  Classifier: Programming Language :: Python :: 3.14
23
22
  Classifier: Topic :: Security :: Cryptography
24
- Requires-Python: >=3.9
23
+ Requires-Python: >=3.10
25
24
  Description-Content-Type: text/markdown
26
25
  License-File: LICENSE
27
26
  Requires-Dist: cryptography>=42
@@ -34,7 +33,7 @@ Requires-Dist: cryptography>=48; extra == "pq"
34
33
  Provides-Extra: pytest
35
34
  Requires-Dist: pytest>=7; extra == "pytest"
36
35
  Provides-Extra: inspect
37
- Requires-Dist: inspect_ai<0.4,>=0.3.112; python_version >= "3.10" and extra == "inspect"
36
+ Requires-Dist: inspect_ai<0.4,>=0.3.112; extra == "inspect"
38
37
  Provides-Extra: dev
39
38
  Requires-Dist: pytest>=7; extra == "dev"
40
39
  Requires-Dist: ruff>=0.5; extra == "dev"
@@ -44,7 +43,7 @@ Requires-Dist: build>=1; extra == "dev"
44
43
  Requires-Dist: hypothesis>=6; extra == "dev"
45
44
  Requires-Dist: rfc8785>=0.1.4; extra == "dev"
46
45
  Requires-Dist: sd-jwt>=0.10; extra == "dev"
47
- Requires-Dist: inspect_ai<0.4,>=0.3.112; python_version >= "3.10" and extra == "dev"
46
+ Requires-Dist: inspect_ai<0.4,>=0.3.112; extra == "dev"
48
47
  Dynamic: license-file
49
48
 
50
49
  <div align="center">
@@ -64,13 +63,16 @@ revocation snapshots, aligned to the in-toto test-result predicate. One portable
64
63
  file, no server, no network.**
65
64
 
66
65
  [![CI](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml/badge.svg)](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
66
+ [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
67
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
68
+ <!-- Badges below are enabled AFTER the first PyPI release (they render broken/false before it,
69
+ which reads as abandonment). Un-comment on first publish — see RELEASE.md:
67
70
  [![PyPI](https://img.shields.io/pypi/v/proofbundle.svg?color=D6248A&cacheSeconds=3600)](https://pypi.org/project/proofbundle/)
68
71
  [![Python](https://img.shields.io/pypi/pyversions/proofbundle.svg?color=D6248A&cacheSeconds=3600)](https://pypi.org/project/proofbundle/)
69
72
  [![Downloads](https://static.pepy.tech/badge/proofbundle)](https://pepy.tech/project/proofbundle)
70
- [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
71
- [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
72
73
  [![SLSA build provenance](https://img.shields.io/badge/SLSA-build_provenance-D6248A.svg)](https://slsa.dev)
73
74
  [![PyPI attestations](https://img.shields.io/badge/PyPI-attestations_(PEP_740)-D6248A.svg)](https://pypi.org/project/proofbundle/)
75
+ -->
74
76
  [![Post-quantum witnesses](https://img.shields.io/badge/PQ-ML--DSA--44_cosignatures-D6248A.svg)](https://github.com/C2SP/C2SP/blob/main/tlog-cosignature.md)
75
77
  [![Mutation tested](https://img.shields.io/badge/tests-mutation_gated-D6248A.svg)](scripts/mutation_check.py)
76
78
  [![C2SP tlog-proof](https://img.shields.io/badge/C2SP-tlog--proof-D6248A.svg)](https://github.com/C2SP/C2SP/blob/main/tlog-proof.md)
@@ -82,7 +84,7 @@ file, no server, no network.**
82
84
 
83
85
  **At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
84
86
  verify` checks one self-contained `bundle.json` with three offline cryptographic
85
- checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 188 tests + a CI mutation gate.
87
+ checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
86
88
 
87
89
  ## Contents
88
90
 
@@ -147,6 +149,8 @@ not a proof that the number is *true* or that the evaluation was well designed.
147
149
  | `reproduced` | independently re-run and matched |
148
150
  | `enclave_attested` | produced in an attested trusted execution environment |
149
151
 
152
+ Since v1.6 `decode_eval_claim(bundle, expected_context=...)` enforces the signed `context_binding` (cross-context replay guard), and verify-side re-checks the `samples.n == n` invariant — guarantees no longer live only in the emitter.
153
+
150
154
  Full detail: **[THREAT_MODEL.md](THREAT_MODEL.md)** — what `verify` catches and what it structurally cannot.
151
155
 
152
156
  ## What it verifies
@@ -159,12 +163,19 @@ A bundle is a single JSON document. `proofbundle` checks, offline:
159
163
  Certificate Transparency)
160
164
  3. **sd-jwt** (optional) — an embedded SD-JWT selective-disclosure credential is
161
165
  well formed, and if an issuer key is given, correctly issuer-signed
162
- 4. **sd-jwt-key-binding** (optional, v1.2) — if the SD-JWT carries a Key Binding
163
- JWT (RFC 9901 §4.3), it is verified **fail-closed**: `typ` is `kb+jwt`,
164
- `iat`/`aud`/`nonce`/`sd_hash` are present, `sd_hash` binds the exact presented
165
- disclosure set, and the signature verifies under the issuer-bound `cnf.jwk`
166
- holder key. A present-but-broken KB-JWT fails the bundle it is never
167
- silently ignored.
166
+ 4. **sd-jwt-key-binding** (optional, v1.2; hardened v1.3) — proof-of-possession, **fail-closed**.
167
+ The check runs only once the issuer signature itself verified (an unauthenticated SD-JWT can
168
+ never report a valid holder binding). Then: `typ` is `kb+jwt`, `iat`/`aud`/`nonce`/`sd_hash` are
169
+ present, `sd_hash` binds the exact presented disclosure set, and the signature verifies under the
170
+ issuer-bound `cnf.jwk` holder key. A present-but-broken KB-JWT fails the bundle; and if the issuer
171
+ **bound** a `cnf` holder key but the presentation carries **no** KB-JWT, that fails too — stripping
172
+ the binding is a bearer downgrade, not a valid receipt. `verify --aud/--nonce` enforce RFC 9901
173
+ §7.3 audience/replay binding when the relying party supplies them.
174
+
175
+ Beyond the single-file bundle, the library also verifies, all offline: a **witnessed C2SP checkpoint**
176
+ (`verify_witnessed_checkpoint`, Ed25519 + post-quantum ML-DSA-44 cosignatures — the quorum counts distinct
177
+ witness **keys**, not names, so one key under many names can never stuff a threshold), a **C2SP
178
+ `.tlog-proof`** (`proofbundle verify-proof`), and a **Token Status List** revocation snapshot.
168
179
 
169
180
  The verifier treats the payload as opaque bytes. It proves that these exact
170
181
  bytes were signed and anchored, not what they mean. That is on purpose: it keeps
@@ -203,7 +214,7 @@ flowchart LR
203
214
  pip install proofbundle
204
215
  ```
205
216
 
206
- Requires Python 3.9+ and [`cryptography`](https://cryptography.io). Signature
217
+ Requires Python 3.10+ and [`cryptography`](https://cryptography.io). Signature
207
218
  math is delegated to `cryptography`; this project never rolls its own crypto.
208
219
  The Merkle and SD-JWT logic is pure standard library.
209
220
 
@@ -217,7 +228,7 @@ pip install "proofbundle[sdjwt]"
217
228
  ## Quickstart
218
229
 
219
230
  ```bash
220
- # generate a real example bundle with throwaway keys
231
+ # from a git checkout (examples/ ships in the repo, not the wheel):
221
232
  python examples/make_example.py
222
233
 
223
234
  # verify it
@@ -228,6 +239,15 @@ proofbundle verify examples/example_bundle.json
228
239
  <img src="assets/demo.svg" alt="proofbundle verify output: PASS checks and OK" width="680">
229
240
  </div>
230
241
 
242
+ See the whole trust story in five seconds — pip-only, offline, in memory (honest receipt verifies, six tampers fail, a swapped sample is caught):
243
+
244
+ ```bash
245
+ pip install proofbundle
246
+ proofbundle demo
247
+ ```
248
+
249
+ Full walkthrough incl. the forced-random-sample audit: **[docs/DEMO.md](docs/DEMO.md)**.
250
+
231
251
  Machine-readable output and a non-zero exit code on failure:
232
252
 
233
253
  ```bash
@@ -321,7 +341,7 @@ Since v1.2 proofbundle also speaks the witness layer:
321
341
  (Ed25519 cosignature/v1) — `verify_witnessed_checkpoint` checks a checkpoint is
322
342
  both log-signed **and** cosigned by a quorum of distinct witnesses, offline,
323
343
  which rules out a split view by the log operator. This is the same
324
- witnessed-checkpoint pattern Rekor v2 (GA October 2025) institutionalizes.
344
+ witnessed-checkpoint pattern Rekor v2 (GA October 2025) is integrating.
325
345
 
326
346
  Since v1.3 the witness layer is complete: **ML-DSA-44 cosignatures** (C2SP type
327
347
  0x06, FIPS 204 — post-quantum, the spec's SHOULD for new witness deployments;
@@ -480,7 +500,7 @@ threshold was met without revealing the model or the data. See [INTEROP.md](INTE
480
500
  [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) `results_*.json` (the
481
501
  genuine `acc,none` filter-suffix format). **`proofbundle.adapters.from_eee_dataset`** (v0.9) reads an
482
502
  Every Eval Ever v0.2.2 aggregate JSON and builds a signed receipt — validated against the vendored EEE
483
- schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.9+).
503
+ schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.10+).
484
504
  - **in-toto test-result export, DSSE-signed** (v0.9) — `proofbundle.intoto.export_intoto_dsse(claim,
485
505
  signer)` emits the receipt as a DSSE-signed in-toto Statement v1 with the **generic
486
506
  `test-result/v0.1` predicate** (result PASSED/FAILED, `configuration` ResourceDescriptors), so a generic
@@ -540,15 +560,33 @@ ISO/IEC DIS 24970), including the anti-patterns no one should claim.
540
560
  (RFC 9901 §4.3, fail-closed; closes #1) with `cnf.jwk` issuance and holder presentation, **C2SP
541
561
  tlog-cosignature** verification (Ed25519 cosignature/v1, witness quorum, split-view resistance), and
542
562
  `verify --verbose` with the recomputed Merkle root (closes #2).
543
- - **v1.3 (current release)** — the portable proof: **C2SP tlog-proof** emit/verify (+
563
+ - **v1.3** — the portable proof: **C2SP tlog-proof** emit/verify (+
544
564
  `verify-proof` CLI), **ML-DSA-44** witness cosignatures (post-quantum, `[pq]` extra), **Token
545
565
  Status List** snapshot verification (offline revocation), SD-JWT VC markers (`dc+sd-jwt`,
546
566
  `vct`, `status`), COMPLIANCE.md (EU AI Act Art. 12 / NIST AI RMF mapping), and the mutation
547
567
  suite as a CI gate.
548
- - **Deferred** (explicitly not yet built) full SD-JWT VC conformance + `vct` type-metadata
549
- resolution (pre-IESG), per-sample Merkle receipts (the THREAT_MODEL's named gap; v2.0
550
- direction), an official in-toto eval predicate (proposal path via OpenSSF/CoSAI), a full
551
- in-toto client, a Python-3.10 floor.
568
+ - **v1.4** — distribution: a **promptfoo** adapter (results.json v3
569
+ pass_rate receipt), the **Hugging Face Community Evals bridge** (`pb1.` receipt tokens for the
570
+ `.eval_results` `verifyToken` field + a strict YAML entry emitter — proofbundle-verifiable,
571
+ explicitly NOT the HF-internal badge token), `hf-token` CLI, and the **Python 3.10+ floor**
572
+ (3.9 is EOL since 2025-10).
573
+ - **v1.5** — per-sample receipts: a signed **Merkle commitment over every
574
+ individual sample** (`samples` claim field, SPEC §7g), salted per-leaf disclosures (RFC 9901
575
+ mechanic), **sample openings** with replay guards, and a **spot-check audit protocol**
576
+ (`audit-challenge` / `verify-opening` CLI; auditor-nonce, beacon, and documented-grindable
577
+ self-challenge modes; PoR soundness 1−(1−m)^k) — narrowing the THREAT_MODEL's sub-sampling gap.
578
+ Extractors for lm-eval `--log_samples` JSONL and promptfoo rows.
579
+ - **v1.6** — external principal-security review fixes: verify-side sample/`leaf_alg`/root invariants
580
+ and signed `context_binding` enforcement (was signed but unchecked), status-list freshness
581
+ (`exp`/`ttl` None-safe), `merkle.hash_alg` now required; Beta classifier; 26-operator mutation gate.
582
+ - **v1.6.1** — developer experience: `proofbundle demo` (pip-only, fail-closed tamper smoke test), a
583
+ runnable per-sample example, `docs/DEMO.md`.
584
+ - **v1.7 (current release)** — release supply-chain hardening: `release.yml` builds once and
585
+ publishes the **attested** bytes (sha256 gate: attested == published), a reviewer-gated `pypi`
586
+ environment, least-privilege permissions, SHA-pinned actions, `RELEASE.md` + `docs/REVIEWERS.md`.
587
+ - **Deferred** (explicitly not yet built, stated honestly) — full SD-JWT VC conformance + `vct`
588
+ type-metadata resolution (pre-IESG), an official in-toto eval predicate (proposal path via
589
+ OpenSSF/CoSAI), and a full in-toto client.
552
590
 
553
591
  ## Contributing
554
592
 
@@ -15,13 +15,16 @@ revocation snapshots, aligned to the in-toto test-result predicate. One portable
15
15
  file, no server, no network.**
16
16
 
17
17
  [![CI](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml/badge.svg)](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
18
+ [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
19
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
20
+ <!-- Badges below are enabled AFTER the first PyPI release (they render broken/false before it,
21
+ which reads as abandonment). Un-comment on first publish — see RELEASE.md:
18
22
  [![PyPI](https://img.shields.io/pypi/v/proofbundle.svg?color=D6248A&cacheSeconds=3600)](https://pypi.org/project/proofbundle/)
19
23
  [![Python](https://img.shields.io/pypi/pyversions/proofbundle.svg?color=D6248A&cacheSeconds=3600)](https://pypi.org/project/proofbundle/)
20
24
  [![Downloads](https://static.pepy.tech/badge/proofbundle)](https://pepy.tech/project/proofbundle)
21
- [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
22
- [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
23
25
  [![SLSA build provenance](https://img.shields.io/badge/SLSA-build_provenance-D6248A.svg)](https://slsa.dev)
24
26
  [![PyPI attestations](https://img.shields.io/badge/PyPI-attestations_(PEP_740)-D6248A.svg)](https://pypi.org/project/proofbundle/)
27
+ -->
25
28
  [![Post-quantum witnesses](https://img.shields.io/badge/PQ-ML--DSA--44_cosignatures-D6248A.svg)](https://github.com/C2SP/C2SP/blob/main/tlog-cosignature.md)
26
29
  [![Mutation tested](https://img.shields.io/badge/tests-mutation_gated-D6248A.svg)](scripts/mutation_check.py)
27
30
  [![C2SP tlog-proof](https://img.shields.io/badge/C2SP-tlog--proof-D6248A.svg)](https://github.com/C2SP/C2SP/blob/main/tlog-proof.md)
@@ -33,7 +36,7 @@ file, no server, no network.**
33
36
 
34
37
  **At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
35
38
  verify` checks one self-contained `bundle.json` with three offline cryptographic
36
- checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 188 tests + a CI mutation gate.
39
+ checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
37
40
 
38
41
  ## Contents
39
42
 
@@ -98,6 +101,8 @@ not a proof that the number is *true* or that the evaluation was well designed.
98
101
  | `reproduced` | independently re-run and matched |
99
102
  | `enclave_attested` | produced in an attested trusted execution environment |
100
103
 
104
+ Since v1.6 `decode_eval_claim(bundle, expected_context=...)` enforces the signed `context_binding` (cross-context replay guard), and verify-side re-checks the `samples.n == n` invariant — guarantees no longer live only in the emitter.
105
+
101
106
  Full detail: **[THREAT_MODEL.md](THREAT_MODEL.md)** — what `verify` catches and what it structurally cannot.
102
107
 
103
108
  ## What it verifies
@@ -110,12 +115,19 @@ A bundle is a single JSON document. `proofbundle` checks, offline:
110
115
  Certificate Transparency)
111
116
  3. **sd-jwt** (optional) — an embedded SD-JWT selective-disclosure credential is
112
117
  well formed, and if an issuer key is given, correctly issuer-signed
113
- 4. **sd-jwt-key-binding** (optional, v1.2) — if the SD-JWT carries a Key Binding
114
- JWT (RFC 9901 §4.3), it is verified **fail-closed**: `typ` is `kb+jwt`,
115
- `iat`/`aud`/`nonce`/`sd_hash` are present, `sd_hash` binds the exact presented
116
- disclosure set, and the signature verifies under the issuer-bound `cnf.jwk`
117
- holder key. A present-but-broken KB-JWT fails the bundle it is never
118
- silently ignored.
118
+ 4. **sd-jwt-key-binding** (optional, v1.2; hardened v1.3) — proof-of-possession, **fail-closed**.
119
+ The check runs only once the issuer signature itself verified (an unauthenticated SD-JWT can
120
+ never report a valid holder binding). Then: `typ` is `kb+jwt`, `iat`/`aud`/`nonce`/`sd_hash` are
121
+ present, `sd_hash` binds the exact presented disclosure set, and the signature verifies under the
122
+ issuer-bound `cnf.jwk` holder key. A present-but-broken KB-JWT fails the bundle; and if the issuer
123
+ **bound** a `cnf` holder key but the presentation carries **no** KB-JWT, that fails too — stripping
124
+ the binding is a bearer downgrade, not a valid receipt. `verify --aud/--nonce` enforce RFC 9901
125
+ §7.3 audience/replay binding when the relying party supplies them.
126
+
127
+ Beyond the single-file bundle, the library also verifies, all offline: a **witnessed C2SP checkpoint**
128
+ (`verify_witnessed_checkpoint`, Ed25519 + post-quantum ML-DSA-44 cosignatures — the quorum counts distinct
129
+ witness **keys**, not names, so one key under many names can never stuff a threshold), a **C2SP
130
+ `.tlog-proof`** (`proofbundle verify-proof`), and a **Token Status List** revocation snapshot.
119
131
 
120
132
  The verifier treats the payload as opaque bytes. It proves that these exact
121
133
  bytes were signed and anchored, not what they mean. That is on purpose: it keeps
@@ -154,7 +166,7 @@ flowchart LR
154
166
  pip install proofbundle
155
167
  ```
156
168
 
157
- Requires Python 3.9+ and [`cryptography`](https://cryptography.io). Signature
169
+ Requires Python 3.10+ and [`cryptography`](https://cryptography.io). Signature
158
170
  math is delegated to `cryptography`; this project never rolls its own crypto.
159
171
  The Merkle and SD-JWT logic is pure standard library.
160
172
 
@@ -168,7 +180,7 @@ pip install "proofbundle[sdjwt]"
168
180
  ## Quickstart
169
181
 
170
182
  ```bash
171
- # generate a real example bundle with throwaway keys
183
+ # from a git checkout (examples/ ships in the repo, not the wheel):
172
184
  python examples/make_example.py
173
185
 
174
186
  # verify it
@@ -179,6 +191,15 @@ proofbundle verify examples/example_bundle.json
179
191
  <img src="assets/demo.svg" alt="proofbundle verify output: PASS checks and OK" width="680">
180
192
  </div>
181
193
 
194
+ See the whole trust story in five seconds — pip-only, offline, in memory (honest receipt verifies, six tampers fail, a swapped sample is caught):
195
+
196
+ ```bash
197
+ pip install proofbundle
198
+ proofbundle demo
199
+ ```
200
+
201
+ Full walkthrough incl. the forced-random-sample audit: **[docs/DEMO.md](docs/DEMO.md)**.
202
+
182
203
  Machine-readable output and a non-zero exit code on failure:
183
204
 
184
205
  ```bash
@@ -272,7 +293,7 @@ Since v1.2 proofbundle also speaks the witness layer:
272
293
  (Ed25519 cosignature/v1) — `verify_witnessed_checkpoint` checks a checkpoint is
273
294
  both log-signed **and** cosigned by a quorum of distinct witnesses, offline,
274
295
  which rules out a split view by the log operator. This is the same
275
- witnessed-checkpoint pattern Rekor v2 (GA October 2025) institutionalizes.
296
+ witnessed-checkpoint pattern Rekor v2 (GA October 2025) is integrating.
276
297
 
277
298
  Since v1.3 the witness layer is complete: **ML-DSA-44 cosignatures** (C2SP type
278
299
  0x06, FIPS 204 — post-quantum, the spec's SHOULD for new witness deployments;
@@ -431,7 +452,7 @@ threshold was met without revealing the model or the data. See [INTEROP.md](INTE
431
452
  [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) `results_*.json` (the
432
453
  genuine `acc,none` filter-suffix format). **`proofbundle.adapters.from_eee_dataset`** (v0.9) reads an
433
454
  Every Eval Ever v0.2.2 aggregate JSON and builds a signed receipt — validated against the vendored EEE
434
- schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.9+).
455
+ schema, with **no runtime import** of `every_eval_ever` (it needs Python 3.12; proofbundle stays 3.10+).
435
456
  - **in-toto test-result export, DSSE-signed** (v0.9) — `proofbundle.intoto.export_intoto_dsse(claim,
436
457
  signer)` emits the receipt as a DSSE-signed in-toto Statement v1 with the **generic
437
458
  `test-result/v0.1` predicate** (result PASSED/FAILED, `configuration` ResourceDescriptors), so a generic
@@ -491,15 +512,33 @@ ISO/IEC DIS 24970), including the anti-patterns no one should claim.
491
512
  (RFC 9901 §4.3, fail-closed; closes #1) with `cnf.jwk` issuance and holder presentation, **C2SP
492
513
  tlog-cosignature** verification (Ed25519 cosignature/v1, witness quorum, split-view resistance), and
493
514
  `verify --verbose` with the recomputed Merkle root (closes #2).
494
- - **v1.3 (current release)** — the portable proof: **C2SP tlog-proof** emit/verify (+
515
+ - **v1.3** — the portable proof: **C2SP tlog-proof** emit/verify (+
495
516
  `verify-proof` CLI), **ML-DSA-44** witness cosignatures (post-quantum, `[pq]` extra), **Token
496
517
  Status List** snapshot verification (offline revocation), SD-JWT VC markers (`dc+sd-jwt`,
497
518
  `vct`, `status`), COMPLIANCE.md (EU AI Act Art. 12 / NIST AI RMF mapping), and the mutation
498
519
  suite as a CI gate.
499
- - **Deferred** (explicitly not yet built) full SD-JWT VC conformance + `vct` type-metadata
500
- resolution (pre-IESG), per-sample Merkle receipts (the THREAT_MODEL's named gap; v2.0
501
- direction), an official in-toto eval predicate (proposal path via OpenSSF/CoSAI), a full
502
- in-toto client, a Python-3.10 floor.
520
+ - **v1.4** — distribution: a **promptfoo** adapter (results.json v3
521
+ pass_rate receipt), the **Hugging Face Community Evals bridge** (`pb1.` receipt tokens for the
522
+ `.eval_results` `verifyToken` field + a strict YAML entry emitter — proofbundle-verifiable,
523
+ explicitly NOT the HF-internal badge token), `hf-token` CLI, and the **Python 3.10+ floor**
524
+ (3.9 is EOL since 2025-10).
525
+ - **v1.5** — per-sample receipts: a signed **Merkle commitment over every
526
+ individual sample** (`samples` claim field, SPEC §7g), salted per-leaf disclosures (RFC 9901
527
+ mechanic), **sample openings** with replay guards, and a **spot-check audit protocol**
528
+ (`audit-challenge` / `verify-opening` CLI; auditor-nonce, beacon, and documented-grindable
529
+ self-challenge modes; PoR soundness 1−(1−m)^k) — narrowing the THREAT_MODEL's sub-sampling gap.
530
+ Extractors for lm-eval `--log_samples` JSONL and promptfoo rows.
531
+ - **v1.6** — external principal-security review fixes: verify-side sample/`leaf_alg`/root invariants
532
+ and signed `context_binding` enforcement (was signed but unchecked), status-list freshness
533
+ (`exp`/`ttl` None-safe), `merkle.hash_alg` now required; Beta classifier; 26-operator mutation gate.
534
+ - **v1.6.1** — developer experience: `proofbundle demo` (pip-only, fail-closed tamper smoke test), a
535
+ runnable per-sample example, `docs/DEMO.md`.
536
+ - **v1.7 (current release)** — release supply-chain hardening: `release.yml` builds once and
537
+ publishes the **attested** bytes (sha256 gate: attested == published), a reviewer-gated `pypi`
538
+ environment, least-privilege permissions, SHA-pinned actions, `RELEASE.md` + `docs/REVIEWERS.md`.
539
+ - **Deferred** (explicitly not yet built, stated honestly) — full SD-JWT VC conformance + `vct`
540
+ type-metadata resolution (pre-IESG), an official in-toto eval predicate (proposal path via
541
+ OpenSSF/CoSAI), and a full in-toto client.
503
542
 
504
543
  ## Contributing
505
544
 
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "proofbundle"
7
- version = "1.3.0"
7
+ version = "1.7.0"
8
8
  description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
9
9
  readme = "README.md"
10
- requires-python = ">=3.9"
10
+ requires-python = ">=3.10"
11
11
  license = { text = "MIT" }
12
12
  authors = [{ name = "Konrad Gruszka" }]
13
13
  keywords = [
@@ -22,11 +22,10 @@ keywords = [
22
22
  "rfc6962",
23
23
  ]
24
24
  classifiers = [
25
- "Development Status :: 3 - Alpha",
25
+ "Development Status :: 4 - Beta",
26
26
  "Intended Audience :: Developers",
27
27
  "License :: OSI Approved :: MIT License",
28
28
  "Programming Language :: Python :: 3",
29
- "Programming Language :: Python :: 3.9",
30
29
  "Programming Language :: Python :: 3.10",
31
30
  "Programming Language :: Python :: 3.11",
32
31
  "Programming Language :: Python :: 3.12",
@@ -56,11 +55,11 @@ pq = ["cryptography>=48"]
56
55
  pytest = ["pytest>=7"]
57
56
  # The inspect_ai adapter uses the STABLE read_eval_log API (lazy import). Pinned with an UPPER bound:
58
57
  # the .eval format + pydantic schema change between versions (inspect_ai issue 834), and the fixture
59
- # test is bound to this range. inspect_ai requires Python >= 3.10, so the marker gates it out on 3.9
60
- # (base + [eval]/[sdjwt] still work on 3.9; the inspect adapter test skips there). Fixes the red 3.9 CI.
61
- inspect = ['inspect_ai>=0.3.112,<0.4; python_version >= "3.10"']
58
+ # test is bound to this range. Since v1.4.0 the project floor is Python 3.10, so no per-extra version
59
+ # marker is needed here (inspect_ai's own >=3.10 requirement is always satisfiable on a supported base).
60
+ inspect = ['inspect_ai>=0.3.112,<0.4']
62
61
  dev = ["pytest>=7", "ruff>=0.5", "jsonschema>=4", "mypy>=1.8", "build>=1", "hypothesis>=6",
63
- "rfc8785>=0.1.4", "sd-jwt>=0.10", 'inspect_ai>=0.3.112,<0.4; python_version >= "3.10"']
62
+ "rfc8785>=0.1.4", "sd-jwt>=0.10", 'inspect_ai>=0.3.112,<0.4']
64
63
 
65
64
  [project.urls]
66
65
  Homepage = "https://b7n0de.com"
@@ -13,7 +13,7 @@ from __future__ import annotations
13
13
 
14
14
  from typing import TYPE_CHECKING
15
15
 
16
- __version__ = "1.3.0"
16
+ __version__ = "1.7.0"
17
17
 
18
18
  __all__ = [
19
19
  "__version__",
@@ -30,6 +30,12 @@ __all__ = [
30
30
  "verify_cosignature",
31
31
  "verify_witnessed_checkpoint",
32
32
  "verify_status_snapshot",
33
+ "receipt_token",
34
+ "verify_receipt_token",
35
+ "build_sample_tree",
36
+ "sample_opening",
37
+ "verify_sample_opening",
38
+ "audit_challenge",
33
39
  "VerificationResult",
34
40
  "Check",
35
41
  "ProofBundleError",
@@ -47,6 +53,12 @@ _LAZY = {
47
53
  "verify_cosignature": ".checkpoint",
48
54
  "verify_witnessed_checkpoint": ".checkpoint",
49
55
  "verify_status_snapshot": ".statuslist",
56
+ "receipt_token": ".hf_evals",
57
+ "verify_receipt_token": ".hf_evals",
58
+ "build_sample_tree": ".persample",
59
+ "sample_opening": ".persample",
60
+ "verify_sample_opening": ".persample",
61
+ "audit_challenge": ".persample",
50
62
  }
51
63
 
52
64
  if TYPE_CHECKING: # static analysers + IDEs see the real names/types; runtime stays lazy
@@ -55,6 +67,9 @@ if TYPE_CHECKING: # static analysers + IDEs see the real names/types; runtime s
55
67
  from .errors import Check, ProofBundleError, VerificationResult
56
68
  from .checkpoint import verify_cosignature, verify_witnessed_checkpoint
57
69
  from .kbjwt import verify_key_binding
70
+ from .hf_evals import receipt_token, verify_receipt_token
71
+ from .persample import (audit_challenge, build_sample_tree, sample_opening,
72
+ verify_sample_opening)
58
73
  from .statuslist import verify_status_snapshot
59
74
  from .tlogproof import verify_tlog_proof
60
75
  from .merkle import verify_consistency, verify_inclusion
@@ -7,5 +7,9 @@ each fixture in tests/fixtures documents its source + version.
7
7
  from .inspect_ai import from_inspect_ai_log
8
8
  from .eee import from_eee_dataset
9
9
  from .lm_eval import from_lm_eval_results
10
+ from .promptfoo import from_promptfoo_results
11
+ from .samples import samples_from_lm_eval_jsonl, samples_from_promptfoo_results
10
12
 
11
- __all__ = ["from_lm_eval_results", "from_inspect_ai_log", "from_eee_dataset"]
13
+ __all__ = ["from_lm_eval_results", "from_inspect_ai_log", "from_eee_dataset",
14
+ "from_promptfoo_results",
15
+ "samples_from_lm_eval_jsonl", "samples_from_promptfoo_results"]
@@ -5,7 +5,7 @@ it has no cryptography. This converter is strictly additive: it reads an EEE agg
5
5
  signed, selectively-disclosable proofbundle receipt from it.
6
6
 
7
7
  IMPORTANT: `every_eval_ever` is NOT imported at runtime — it requires Python 3.12+ (pydantic/numpy/pandas/
8
- duckdb), while proofbundle stays 3.9+. We parse the EEE JSON directly and OPTIONALLY validate it against the
8
+ duckdb), while proofbundle stays 3.10+. We parse the EEE JSON directly and OPTIONALLY validate it against the
9
9
  vendored `eee_eval_schema.json` (schema version 0.2.2, MIT) using `jsonschema` if available.
10
10
 
11
11
  Field mapping (verified 2026-07 against schemas/eval.schema.json v0.2.2):
@@ -0,0 +1,131 @@
1
+ """Adapter for promptfoo `eval -o results.json` output (file-based, NO promptfoo import) — v1.4.
2
+
3
+ Parses the exported OutputFile JSON only. Format verified 2026-07-02 against promptfoo `main`
4
+ (`src/types/index.ts` OutputFile / EvaluateSummaryV3 / EvaluateStats, `src/util/output.ts`
5
+ writeOutput): top level carries `evalId`, `results`, `config`, `metadata`; the current summary is
6
+ **version 3** (`results.version == 3`) with whole-run aggregates in `results.stats` as
7
+ `successes` / `failures` / `errors`. The metric emitted is **pass_rate** = successes / (successes
8
+ + failures + errors), rendered as a fixed-point decimal string (the claim schema takes decimal
9
+ strings, never floats).
10
+
11
+ Version honesty: legacy files with `results.version` 1/2 have a different shape (`table` instead
12
+ of `prompts`) and are REJECTED with a clear message rather than half-parsed; the "v4" storage
13
+ version never appears in output files (they still say `version: 3`). Redaction flags
14
+ (`PROMPTFOO_STRIP_*`) only affect prompt/response bodies — nothing this adapter reads.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import hashlib
19
+ import json
20
+ from decimal import Decimal
21
+ from pathlib import Path
22
+ from typing import Optional
23
+
24
+ from ..evalclaim import build_eval_claim
25
+
26
+ _SCALE = 6 # pass_rate decimal places — fixed-point, schema-conformant
27
+
28
+
29
+ def _pass_rate(successes: int, failures: int, errors: int) -> "tuple[str, int]":
30
+ total = successes + failures + errors
31
+ if total <= 0:
32
+ raise ValueError("promptfoo results contain no test outcomes (stats all zero)")
33
+ rate = (Decimal(successes) / Decimal(total)).quantize(Decimal(1).scaleb(-_SCALE))
34
+ return f"{rate:.{_SCALE}f}", total
35
+
36
+
37
+ def from_promptfoo_results(path, *, comparator: str, threshold: str, timestamp: str,
38
+ model_salt: Optional[bytes] = None,
39
+ dataset_salt: Optional[bytes] = None):
40
+ """Read a promptfoo results.json (summary v3) and build an eval claim over the run's
41
+ pass_rate. Returns (claim, salts).
42
+
43
+ - suite = `config.description` when present, else the `evalId` (the run's identity).
44
+ - model_id = the sorted, de-duplicated provider ids of the run (a promptfoo eval may span
45
+ several providers; the salted commitment pins the exact set).
46
+ - dataset_id = sha256 over the canonical JSON of `config.tests` AS RECORDED in results.json.
47
+ Inline tests bind the actual content (the suite IS the dataset); a `file://`/glob reference binds
48
+ the reference, not the resolved file content (an offline reader cannot resolve it) — provenance
49
+ records which case applies (`dataset_commitment_scope`) so the binding is never overstated.
50
+ - provenance: promptfooVersion, evalId, summary timestamp, per-outcome counts.
51
+ """
52
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
53
+ if not isinstance(data, dict) or not isinstance(data.get("results"), dict):
54
+ raise ValueError("not a promptfoo output file (missing results object)")
55
+ summary = data["results"]
56
+ version = summary.get("version")
57
+ if version != 3:
58
+ raise ValueError(
59
+ f"unsupported promptfoo summary version {version!r} — this adapter parses the "
60
+ "current version 3 (legacy v1/v2 files use a different 'table' shape; re-export "
61
+ "with a current promptfoo)")
62
+ stats = summary.get("stats")
63
+ if not isinstance(stats, dict):
64
+ raise ValueError("promptfoo results.stats missing")
65
+ counts = {}
66
+ for key in ("successes", "failures", "errors"):
67
+ val = stats.get(key, 0)
68
+ if isinstance(val, bool) or not isinstance(val, int) or val < 0:
69
+ raise ValueError(f"promptfoo stats.{key} must be a non-negative integer")
70
+ counts[key] = val
71
+ score, n = _pass_rate(counts["successes"], counts["failures"], counts["errors"])
72
+
73
+ config = data.get("config") or {}
74
+ eval_id = data.get("evalId")
75
+ suite = str(config.get("description") or eval_id or "promptfoo-run")
76
+
77
+ # Commit to the providers that ACTUALLY produced a result in this run — not config.providers, which can
78
+ # list providers that never ran (e.g. --filter-providers). Only fall back to config.providers if the
79
+ # summary recorded no per-result provider at all (release-review fix).
80
+ providers = set()
81
+ for res in summary.get("results", []):
82
+ prov = (res or {}).get("provider")
83
+ if isinstance(prov, dict) and prov.get("id"):
84
+ providers.add(str(prov["id"]))
85
+ elif isinstance(prov, str):
86
+ providers.add(prov)
87
+ if not providers:
88
+ for prov in config.get("providers") or []:
89
+ if isinstance(prov, str):
90
+ providers.add(prov)
91
+ elif isinstance(prov, dict) and prov.get("id"):
92
+ providers.add(str(prov["id"]))
93
+ model_id = "+".join(sorted(providers)) if providers else "unknown:promptfoo-provider"
94
+
95
+ tests = config.get("tests")
96
+ tests_canonical = json.dumps(tests, sort_keys=True, separators=(",", ":")) if tests else ""
97
+ dataset_id = ("promptfoo-tests-sha256:"
98
+ + hashlib.sha256(tests_canonical.encode("utf-8")).hexdigest())
99
+ # HONESTY (release-review): the commitment is over `config.tests` AS RECORDED in results.json. When tests are
100
+ # INLINE, that is the content (the suite IS the dataset). When they are a `file://`/glob REFERENCE — whether a
101
+ # top-level string, a list containing a string, OR a `file://` nested inside an inline test's vars/assert —
102
+ # promptfoo records the unresolved reference, so the commitment binds the reference, NOT the file content
103
+ # (an offline reader cannot resolve external files). When there are NO tests at all the commitment is over the
104
+ # empty string and binds nothing. We label each case so a verifier is never misled that content was bound.
105
+ tests_by_reference = isinstance(tests, str) or (
106
+ isinstance(tests, list) and any(isinstance(t, str) for t in tests)) or ("file://" in tests_canonical)
107
+ if not tests:
108
+ commitment_scope = "config.tests_absent"
109
+ elif tests_by_reference:
110
+ commitment_scope = "config.tests_reference_only"
111
+ else:
112
+ commitment_scope = "config.tests_inline_content"
113
+
114
+ metadata = data.get("metadata") or {}
115
+ provenance = {"harness": "promptfoo",
116
+ "successes": str(counts["successes"]), "failures": str(counts["failures"]),
117
+ "errors": str(counts["errors"]),
118
+ "pass_rate_formula": "successes/(successes+failures+errors)",
119
+ "dataset_commitment_scope": commitment_scope}
120
+ if eval_id:
121
+ provenance["eval_id"] = str(eval_id)
122
+ if metadata.get("promptfooVersion"):
123
+ provenance["promptfoo_version"] = str(metadata["promptfooVersion"])
124
+ if summary.get("timestamp"):
125
+ provenance["run_timestamp"] = str(summary["timestamp"])
126
+
127
+ return build_eval_claim(
128
+ suite=suite, suite_version=f"promptfoo-summary-v{version}",
129
+ metric="pass_rate", comparator=comparator, threshold=threshold, score=score, n=n,
130
+ model_id=model_id, dataset_id=dataset_id, issuer="", timestamp=timestamp,
131
+ provenance=provenance, model_salt=model_salt, dataset_salt=dataset_salt)