proofbundle 1.9.1__tar.gz → 1.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {proofbundle-1.9.1/src/proofbundle.egg-info → proofbundle-1.9.2}/PKG-INFO +20 -8
  2. {proofbundle-1.9.1 → proofbundle-1.9.2}/README.md +19 -7
  3. {proofbundle-1.9.1 → proofbundle-1.9.2}/pyproject.toml +1 -1
  4. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/__init__.py +1 -1
  5. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/bundle.py +14 -0
  6. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/evalclaim.py +8 -0
  7. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/statuslist.py +1 -1
  8. {proofbundle-1.9.1 → proofbundle-1.9.2/src/proofbundle.egg-info}/PKG-INFO +20 -8
  9. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle.egg-info/SOURCES.txt +2 -0
  10. proofbundle-1.9.2/tests/test_claims_hygiene.py +48 -0
  11. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_cli_eval.py +25 -0
  12. proofbundle-1.9.2/tests/test_docs_truth.py +46 -0
  13. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_evalclaim.py +23 -0
  14. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_kbjwt.py +16 -0
  15. {proofbundle-1.9.1 → proofbundle-1.9.2}/LICENSE +0 -0
  16. {proofbundle-1.9.1 → proofbundle-1.9.2}/setup.cfg +0 -0
  17. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/_inspect_registry.py +0 -0
  18. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/_integration.py +0 -0
  19. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/__init__.py +0 -0
  20. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/_provenance.py +0 -0
  21. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/eee.py +0 -0
  22. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/inspect_ai.py +0 -0
  23. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/lm_eval.py +0 -0
  24. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/promptfoo.py +0 -0
  25. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/adapters/samples.py +0 -0
  26. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/beacon.py +0 -0
  27. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/checkpoint.py +0 -0
  28. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/cli.py +0 -0
  29. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/demo.py +0 -0
  30. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/dsse.py +0 -0
  31. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/eee_eval_schema.json +0 -0
  32. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/emit.py +0 -0
  33. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/errors.py +0 -0
  34. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/hf_evals.py +0 -0
  35. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/inspect_hook.py +0 -0
  36. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/intoto.py +0 -0
  37. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/kbjwt.py +0 -0
  38. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/merkle.py +0 -0
  39. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/persample.py +0 -0
  40. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/prereg.py +0 -0
  41. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/py.typed +0 -0
  42. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/pytest_plugin.py +0 -0
  43. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/sdjwt.py +0 -0
  44. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/sdjwt_issue.py +0 -0
  45. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/signature.py +0 -0
  46. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle/tlogproof.py +0 -0
  47. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle.egg-info/dependency_links.txt +0 -0
  48. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle.egg-info/entry_points.txt +0 -0
  49. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle.egg-info/requires.txt +0 -0
  50. {proofbundle-1.9.1 → proofbundle-1.9.2}/src/proofbundle.egg-info/top_level.txt +0 -0
  51. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_adapters.py +0 -0
  52. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_adversarial.py +0 -0
  53. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_beacon.py +0 -0
  54. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_bundle.py +0 -0
  55. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_bundle_robustness.py +0 -0
  56. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_checkpoint.py +0 -0
  57. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_cli.py +0 -0
  58. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_cosignature.py +0 -0
  59. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_cosignature_mldsa.py +0 -0
  60. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_demo.py +0 -0
  61. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_eee.py +0 -0
  62. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_emit.py +0 -0
  63. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_eval_claim_schema.py +0 -0
  64. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_examples.py +0 -0
  65. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_fuzz_parsers.py +0 -0
  66. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_hf_evals.py +0 -0
  67. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_inspect_hook.py +0 -0
  68. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_intoto.py +0 -0
  69. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_intoto_dsse.py +0 -0
  70. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_merkle.py +0 -0
  71. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_merkle_property.py +0 -0
  72. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_persample.py +0 -0
  73. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_prereg.py +0 -0
  74. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_promptfoo.py +0 -0
  75. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_provenance.py +0 -0
  76. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_pytest_plugin.py +0 -0
  77. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_rekor_interop.py +0 -0
  78. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_rfc6962_external_vectors.py +0 -0
  79. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_schema.py +0 -0
  80. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_sdjwt_issue.py +0 -0
  81. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_sdjwt_reference.py +0 -0
  82. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_signature.py +0 -0
  83. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_statuslist.py +0 -0
  84. {proofbundle-1.9.1 → proofbundle-1.9.2}/tests/test_tlogproof.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: proofbundle
3
- Version: 1.9.1
3
+ Version: 1.9.2
4
4
  Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
5
5
  Author: Konrad Gruszka
6
6
  License: MIT
@@ -55,9 +55,11 @@ Dynamic: license-file
55
55
 
56
56
  <h1>proofbundle</h1>
57
57
 
58
- **Turn an AI eval result into one portable, offline-verifiable receipt.**
59
- It proves *who signed these exact bytes* and *that nothing changed since* — not that the number is
60
- true. Ed25519 + RFC 6962 Merkle, one file, no server, no network.
58
+ **AI eval results need receipts.**
59
+
60
+ Turn an AI evaluation result into one portable, offline-verifiable receipt. It proves *who signed
61
+ these exact bytes* and *that nothing changed since* — not that the number is true. Ed25519 + RFC 6962
62
+ Merkle, one file, no server, no network.
61
63
 
62
64
  [![CI](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml/badge.svg)](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
63
65
  [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
@@ -108,6 +110,13 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
108
110
  and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
109
111
  **[THREAT_MODEL.md](THREAT_MODEL.md)**.
110
112
 
113
+ ## In plain language
114
+
115
+ A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
116
+ number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
117
+ cash-register receipt does not show the meal was good — but without a receipt there is nothing to
118
+ check at all.
119
+
111
120
  ## How it fits together
112
121
 
113
122
  ```mermaid
@@ -132,13 +141,16 @@ flowchart LR
132
141
  - **Transparency-log interop** — C2SP `tlog-checkpoint` / cosignature / `.tlog-proof`, with
133
142
  post-quantum **ML-DSA-44** witness cosignatures. Optional Token-Status-List revocation snapshots.
134
143
  - **Per-sample audit** — commit to every sample; an auditor challenges random indices (with a fresh
135
- nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. Catches
136
- 1% sample-doctoring with 95% confidence at 300 samples, regardless of run size.
144
+ nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. With
145
+ such an auditor-supplied or beacon-bound challenge, 300 samples catch 1% sample-doctoring with 95%
146
+ confidence, regardless of run size — a challenge the issuer chose itself does not give this
147
+ guarantee.
137
148
  - **Pre-registration** — `proofbundle prereg <plan>` commits to the protocol before the run, so
138
149
  best-of-many publishing becomes visible.
139
150
  - **Integrations** — opt-in inspect_ai end-of-task hook and pytest plugin (emit only when
140
151
  `PROOFBUNDLE_EMIT=1` / `--proofbundle`), plus a Hugging Face Community Evals bridge. See
141
- [INTEGRATIONS.md](INTEGRATIONS.md).
152
+ [INTEGRATIONS.md](INTEGRATIONS.md), or the end-to-end walkthrough
153
+ [docs/INSPECT_HAPPY_PATH.md](docs/INSPECT_HAPPY_PATH.md) — run an eval, get a receipt, verify it offline.
142
154
 
143
155
  ## Docs
144
156
 
@@ -168,7 +180,7 @@ Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 co
168
180
 
169
181
  ## Status & scope
170
182
 
171
- Beta, SemVer-committed, 303 tests + a CI mutation gate + property-based parser fuzzing. Correctness
183
+ Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
172
184
  is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
173
185
  **not** a log service, a full in-toto client, a TEE, a consensus network, or a compliance product
174
186
  by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
@@ -7,9 +7,11 @@
7
7
 
8
8
  <h1>proofbundle</h1>
9
9
 
10
- **Turn an AI eval result into one portable, offline-verifiable receipt.**
11
- It proves *who signed these exact bytes* and *that nothing changed since* — not that the number is
12
- true. Ed25519 + RFC 6962 Merkle, one file, no server, no network.
10
+ **AI eval results need receipts.**
11
+
12
+ Turn an AI evaluation result into one portable, offline-verifiable receipt. It proves *who signed
13
+ these exact bytes* and *that nothing changed since* — not that the number is true. Ed25519 + RFC 6962
14
+ Merkle, one file, no server, no network.
13
15
 
14
16
  [![CI](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml/badge.svg)](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
15
17
  [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
@@ -60,6 +62,13 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
60
62
  and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
61
63
  **[THREAT_MODEL.md](THREAT_MODEL.md)**.
62
64
 
65
+ ## In plain language
66
+
67
+ A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
68
+ number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
69
+ cash-register receipt does not show the meal was good — but without a receipt there is nothing to
70
+ check at all.
71
+
63
72
  ## How it fits together
64
73
 
65
74
  ```mermaid
@@ -84,13 +93,16 @@ flowchart LR
84
93
  - **Transparency-log interop** — C2SP `tlog-checkpoint` / cosignature / `.tlog-proof`, with
85
94
  post-quantum **ML-DSA-44** witness cosignatures. Optional Token-Status-List revocation snapshots.
86
95
  - **Per-sample audit** — commit to every sample; an auditor challenges random indices (with a fresh
87
- nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. Catches
88
- 1% sample-doctoring with 95% confidence at 300 samples, regardless of run size.
96
+ nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. With
97
+ such an auditor-supplied or beacon-bound challenge, 300 samples catch 1% sample-doctoring with 95%
98
+ confidence, regardless of run size — a challenge the issuer chose itself does not give this
99
+ guarantee.
89
100
  - **Pre-registration** — `proofbundle prereg <plan>` commits to the protocol before the run, so
90
101
  best-of-many publishing becomes visible.
91
102
  - **Integrations** — opt-in inspect_ai end-of-task hook and pytest plugin (emit only when
92
103
  `PROOFBUNDLE_EMIT=1` / `--proofbundle`), plus a Hugging Face Community Evals bridge. See
93
- [INTEGRATIONS.md](INTEGRATIONS.md).
104
+ [INTEGRATIONS.md](INTEGRATIONS.md), or the end-to-end walkthrough
105
+ [docs/INSPECT_HAPPY_PATH.md](docs/INSPECT_HAPPY_PATH.md) — run an eval, get a receipt, verify it offline.
94
106
 
95
107
  ## Docs
96
108
 
@@ -120,7 +132,7 @@ Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 co
120
132
 
121
133
  ## Status & scope
122
134
 
123
- Beta, SemVer-committed, 303 tests + a CI mutation gate + property-based parser fuzzing. Correctness
135
+ Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
124
136
  is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
125
137
  **not** a log service, a full in-toto client, a TEE, a consensus network, or a compliance product
126
138
  by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "proofbundle"
7
- version = "1.9.1"
7
+ version = "1.9.2"
8
8
  description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -13,7 +13,7 @@ from __future__ import annotations
13
13
 
14
14
  from typing import TYPE_CHECKING
15
15
 
16
- __version__ = "1.9.1"
16
+ __version__ = "1.9.2"
17
17
 
18
18
  __all__ = [
19
19
  "__version__",
@@ -153,6 +153,7 @@ def verify_bundle(bundle: Union[dict, str], *, expected_aud=None, expected_nonce
153
153
 
154
154
  # 3. optional SD-JWT selective disclosure credential
155
155
  sd = bundle.get("sd_jwt_vc")
156
+ kb_binding_checked = False # F4: did a KB-JWT (the aud/nonce carrier) actually get verified?
156
157
  if sd is not None:
157
158
  sd = _require_dict(sd, "sd_jwt_vc")
158
159
  _reject_unknown(sd, _SD_KEYS, "sd_jwt_vc")
@@ -186,6 +187,7 @@ def verify_bundle(bundle: Union[dict, str], *, expected_aud=None, expected_nonce
186
187
  if kb is not None:
187
188
  kb_res = verify_key_binding(compact, expected_aud=expected_aud, expected_nonce=expected_nonce)
188
189
  result.add("sd-jwt-key-binding", kb_res["ok"], kb_res["detail"])
190
+ kb_binding_checked = True
189
191
  elif _issuer_requires_holder_binding(sd_part):
190
192
  result.add(
191
193
  "sd-jwt-key-binding", False,
@@ -206,6 +208,18 @@ def verify_bundle(bundle: Union[dict, str], *, expected_aud=None, expected_nonce
206
208
  "binding is unverifiable, refusing (fail-closed; supply "
207
209
  "sd_jwt_vc.issuer_public_key_b64)")
208
210
 
211
+ # F4 (v1.9.2, fail-closed): supplying expected_aud/expected_nonce asks for RFC 9901 §7.3
212
+ # replay/audience binding. A bundle with no verifiable KB-JWT (no sd_jwt_vc at all, or an
213
+ # sd_jwt_vc without a Key Binding JWT) carries nothing to bind to — returning OK anyway is a
214
+ # downgrade trap: the verifier believes the presentation was bound to its aud/nonce when it was
215
+ # not. Refuse the binding it asked for but cannot be enforced. Verifiers that pass no expected_*
216
+ # are unaffected (backward-compatible: the check only fires when a binding was actually requested).
217
+ if (expected_aud is not None or expected_nonce is not None) and not kb_binding_checked:
218
+ result.add(
219
+ "sd-jwt-key-binding", False,
220
+ "expected_aud/expected_nonce were supplied but the bundle carries no verifiable Key "
221
+ "Binding JWT — the requested replay/audience binding cannot be enforced (fail-closed)")
222
+
209
223
  return result
210
224
 
211
225
 
@@ -269,6 +269,14 @@ def decode_eval_claim(bundle, *, expected_context: Optional[str] = None) -> Opti
269
269
  claim = load_claim_text(payload.decode("utf-8"))
270
270
  if claim.get("schema") != EVAL_CLAIM_SCHEMA:
271
271
  return None
272
+ # F3 (v1.9.2): the exact key set is a VERIFY-path invariant, not only an emit-side one.
273
+ # emit_eval_receipt enforces _REQUIRED/_OPTIONAL at emit, but a hand-signed claim bypasses
274
+ # that path — a decoded claim missing a required field or carrying an unknown one was
275
+ # previously ACCEPTED here (the emit-vs-verify asymmetry class this project documents; the
276
+ # module comment on _REQUIRED, "decode/validate reject anything else", was aspirational on
277
+ # this path). Reject fail-closed, mirroring the emit-side check.
278
+ if (_REQUIRED - set(claim)) or (set(claim) - _REQUIRED - _OPTIONAL):
279
+ return None
272
280
  # Issuer binding: the claim's issuer must be the key that signed the bundle.
273
281
  sig_pub_b64 = bundle["signature"]["public_key_b64"]
274
282
  want = "ed25519:" + base64.b64encode(base64.b64decode(sig_pub_b64)).decode("ascii")
@@ -92,7 +92,7 @@ def verify_status_snapshot(status_list_token: str, *, expected_uri: str, index:
92
92
  covers signature + structure + lookup; combining ``ok`` with ``fresh``/``self_issued`` is the
93
93
  caller's policy.
94
94
  """
95
- result = {"ok": False, "status": None, "status_label": None, "fresh": None,
95
+ result: dict[str, object] = {"ok": False, "status": None, "status_label": None, "fresh": None,
96
96
  "self_issued": None, "iat": None, "exp": None, "ttl": None, "detail": ""}
97
97
  if receipt_issuer_pubkey is not None:
98
98
  # hmac.compare_digest for a constant-time compare of the two public keys (defensive; the
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: proofbundle
3
- Version: 1.9.1
3
+ Version: 1.9.2
4
4
  Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
5
5
  Author: Konrad Gruszka
6
6
  License: MIT
@@ -55,9 +55,11 @@ Dynamic: license-file
55
55
 
56
56
  <h1>proofbundle</h1>
57
57
 
58
- **Turn an AI eval result into one portable, offline-verifiable receipt.**
59
- It proves *who signed these exact bytes* and *that nothing changed since* — not that the number is
60
- true. Ed25519 + RFC 6962 Merkle, one file, no server, no network.
58
+ **AI eval results need receipts.**
59
+
60
+ Turn an AI evaluation result into one portable, offline-verifiable receipt. It proves *who signed
61
+ these exact bytes* and *that nothing changed since* — not that the number is true. Ed25519 + RFC 6962
62
+ Merkle, one file, no server, no network.
61
63
 
62
64
  [![CI](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml/badge.svg)](https://github.com/b7n0de/proofbundle/actions/workflows/ci.yml)
63
65
  [![License: MIT](https://img.shields.io/badge/license-MIT-D6248A.svg)](LICENSE)
@@ -108,6 +110,13 @@ This boundary is the point, not a weakness. A receipt makes a claim **attributab
108
110
  and — with pre-registration and per-sample auditing — bounded and spot-checkable**. Full detail:
109
111
  **[THREAT_MODEL.md](THREAT_MODEL.md)**.
110
112
 
113
+ ## In plain language
114
+
115
+ A proofbundle receipt is the cash-register receipt of an AI test result: it shows who claimed the
116
+ number and that nobody quietly changed it afterwards. It does not show the test was good — the way a
117
+ cash-register receipt does not show the meal was good — but without a receipt there is nothing to
118
+ check at all.
119
+
111
120
  ## How it fits together
112
121
 
113
122
  ```mermaid
@@ -132,13 +141,16 @@ flowchart LR
132
141
  - **Transparency-log interop** — C2SP `tlog-checkpoint` / cosignature / `.tlog-proof`, with
133
142
  post-quantum **ML-DSA-44** witness cosignatures. Optional Token-Status-List revocation snapshots.
134
143
  - **Per-sample audit** — commit to every sample; an auditor challenges random indices (with a fresh
135
- nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. Catches
136
- 1% sample-doctoring with 95% confidence at 300 samples, regardless of run size.
144
+ nonce or a **public randomness beacon**, v1.9) and openings must bind to the signed root. With
145
+ such an auditor-supplied or beacon-bound challenge, 300 samples catch 1% sample-doctoring with 95%
146
+ confidence, regardless of run size — a challenge the issuer chose itself does not give this
147
+ guarantee.
137
148
  - **Pre-registration** — `proofbundle prereg <plan>` commits to the protocol before the run, so
138
149
  best-of-many publishing becomes visible.
139
150
  - **Integrations** — opt-in inspect_ai end-of-task hook and pytest plugin (emit only when
140
151
  `PROOFBUNDLE_EMIT=1` / `--proofbundle`), plus a Hugging Face Community Evals bridge. See
141
- [INTEGRATIONS.md](INTEGRATIONS.md).
152
+ [INTEGRATIONS.md](INTEGRATIONS.md), or the end-to-end walkthrough
153
+ [docs/INSPECT_HAPPY_PATH.md](docs/INSPECT_HAPPY_PATH.md) — run an eval, get a receipt, verify it offline.
142
154
 
143
155
  ## Docs
144
156
 
@@ -168,7 +180,7 @@ Requires Python 3.10+. The verify path never rolls its own crypto — Ed25519 co
168
180
 
169
181
  ## Status & scope
170
182
 
171
- Beta, SemVer-committed, 303 tests + a CI mutation gate + property-based parser fuzzing. Correctness
183
+ Beta, SemVer-committed, with a CI test suite behind a mutation gate + property-based parser fuzzing. Correctness
172
184
  is anchored to external RFC 6962 vectors and a real Rekor proof, not just its own bundles. It is
173
185
  **not** a log service, a full in-toto client, a TEE, a consensus network, or a compliance product
174
186
  by itself — it is the small, offline, standards-native receipt layer between them. Security policy:
@@ -47,11 +47,13 @@ tests/test_beacon.py
47
47
  tests/test_bundle.py
48
48
  tests/test_bundle_robustness.py
49
49
  tests/test_checkpoint.py
50
+ tests/test_claims_hygiene.py
50
51
  tests/test_cli.py
51
52
  tests/test_cli_eval.py
52
53
  tests/test_cosignature.py
53
54
  tests/test_cosignature_mldsa.py
54
55
  tests/test_demo.py
56
+ tests/test_docs_truth.py
55
57
  tests/test_eee.py
56
58
  tests/test_emit.py
57
59
  tests/test_eval_claim_schema.py
@@ -0,0 +1,48 @@
1
+ """Claims-hygiene gate — the docs must not carry un-negated overclaims (six-lens review §15)."""
2
+ import importlib.util
3
+ import tempfile
4
+ import unittest
5
+ from pathlib import Path
6
+
7
+ REPO = Path(__file__).resolve().parents[1]
8
+ _spec = importlib.util.spec_from_file_location("claims_hygiene_check", REPO / "scripts" / "claims_hygiene_check.py")
9
+ ch = importlib.util.module_from_spec(_spec)
10
+ _spec.loader.exec_module(ch)
11
+
12
+
13
+ class TestClaimsHygiene(unittest.TestCase):
14
+ def test_real_docs_are_clean(self):
15
+ # The shipped docs must pass the gate — this is the tool whose product is honest scoping.
16
+ self.assertEqual(ch.main(["--json"]), 0, "the repository docs carry an un-negated overclaim")
17
+
18
+ def test_un_negated_forbidden_phrase_is_a_violation(self):
19
+ with tempfile.TemporaryDirectory() as d:
20
+ for text in ("proofbundle proves correctness of the eval.",
21
+ "The receipt is audit-proof.",
22
+ "It is quantum-safe.",
23
+ "This is the industry standard for receipts.",
24
+ "It is compliance ready today."):
25
+ p = Path(d) / "x.md"
26
+ p.write_text(text, encoding="utf-8")
27
+ self.assertTrue(ch.scan_file(p), f"must flag: {text!r}")
28
+
29
+ def test_negated_forbidden_phrase_is_allowed(self):
30
+ with tempfile.TemporaryDirectory() as d:
31
+ for text in ("A receipt does not prove correctness.",
32
+ "proofbundle is not quantum-safe (Ed25519).",
33
+ "It never proves the number is true.",
34
+ "This is not audit-proof and makes no compliance claim."):
35
+ p = Path(d) / "x.md"
36
+ p.write_text(text, encoding="utf-8")
37
+ self.assertEqual(ch.scan_file(p), [], f"negated form must be allowed: {text!r}")
38
+
39
+ def test_code_fence_and_inline_code_skipped(self):
40
+ with tempfile.TemporaryDirectory() as d:
41
+ p = Path(d) / "x.md"
42
+ p.write_text("```\nproves correctness\n```\nUse `quantum-safe` as a literal flag name.",
43
+ encoding="utf-8")
44
+ self.assertEqual(ch.scan_file(p), [], "code fences / inline code are not prose")
45
+
46
+
47
+ if __name__ == "__main__":
48
+ unittest.main()
@@ -34,6 +34,31 @@ class TestCliEval(unittest.TestCase):
34
34
  self.assertEqual(show.returncode, 0)
35
35
  self.assertIn("passed", show.stdout)
36
36
 
37
+ def test_show_eval_malformed_no_raw_traceback(self):
38
+ # F3 (v1.9.2): show-eval honors the no-raw-traceback contract. A hand-signed receipt missing a
39
+ # required field is now rejected by decode_eval_claim (SH1), so show-eval prints a clean FAILED
40
+ # and exits 1 — it never reaches the claim[<required>] field access that previously raised a
41
+ # raw KeyError outside the try block.
42
+ import os
43
+ import tempfile
44
+
45
+ from proofbundle import evalclaim as ec
46
+ from proofbundle.emit import emit_bundle, generate_signer
47
+ signer = generate_signer()
48
+ c, _ = ec.build_eval_claim(
49
+ suite="s", suite_version="v1", metric="acc", comparator=">=", threshold="0.80",
50
+ score="0.90", n=100, model_id="m", dataset_id="d", issuer="x",
51
+ timestamp="2026-07-01T12:00:00Z", model_salt=b"0" * 16, dataset_salt=b"1" * 16)
52
+ good = ec.decode_eval_claim(ec.emit_eval_receipt(c, signer))
53
+ malformed = {k: v for k, v in good.items() if k != "suite"} # drop a required field, re-sign by hand
54
+ with tempfile.TemporaryDirectory() as d:
55
+ p = os.path.join(d, "bad.json")
56
+ Path(p).write_text(json.dumps(emit_bundle(ec.canonicalize(malformed), signer)), encoding="utf-8")
57
+ r = _run("show-eval", p)
58
+ self.assertEqual(r.returncode, 1, "malformed receipt must exit 1, not crash")
59
+ self.assertNotIn("Traceback", r.stderr, "show-eval must never emit a raw traceback")
60
+ self.assertIn("FAILED", r.stderr)
61
+
37
62
 
38
63
  if __name__ == "__main__":
39
64
  unittest.main()
@@ -0,0 +1,46 @@
1
+ """Doc-truth guards — metrics in the docs must not be able to go stale (six-lens review F5/SH4)."""
2
+ import importlib.util
3
+ import re
4
+ import unittest
5
+ from pathlib import Path
6
+
7
+ REPO = Path(__file__).resolve().parents[1]
8
+ _dl_spec = importlib.util.spec_from_file_location("doc_link_check", REPO / "scripts" / "doc_link_check.py")
9
+ doc_link_check = importlib.util.module_from_spec(_dl_spec)
10
+ _dl_spec.loader.exec_module(doc_link_check)
11
+
12
+
13
+ class TestDocsTruth(unittest.TestCase):
14
+ def test_readme_carries_no_hardcoded_test_count(self):
15
+ # F5: the README stated "303 tests" while the suite had grown past it. A hardcoded count
16
+ # goes stale on every added test. Removed (not tracked by hand) — this guard keeps it gone:
17
+ # a "<N> tests" phrase in the README is a stale-metric regression.
18
+ readme = (REPO / "README.md").read_text(encoding="utf-8")
19
+ # ignore fenced code blocks (a CLI sample line is not a prose metric)
20
+ prose = re.sub(r"```.*?```", "", readme, flags=re.DOTALL)
21
+ hits = re.findall(r"\b\d+\s+tests?\b", prose, flags=re.IGNORECASE)
22
+ self.assertEqual(hits, [], f"README carries a hardcoded, stale-prone test count: {hits}")
23
+
24
+ def test_citation_version_matches_pyproject(self):
25
+ # F6/SH4: CITATION.cff stated version 0.7.0 while pyproject shipped 1.9.1 — a stale version
26
+ # travels into every citation. Pin them together so a release bump cannot drift them apart
27
+ # (the RELEASE.md checklist requires bumping both).
28
+ cff = (REPO / "CITATION.cff").read_text(encoding="utf-8")
29
+ pyproject = (REPO / "pyproject.toml").read_text(encoding="utf-8")
30
+ cff_v = re.search(r"(?m)^version:\s*([^\s#]+)", cff)
31
+ py_v = re.search(r'(?m)^version\s*=\s*"([^"]+)"', pyproject)
32
+ self.assertIsNotNone(cff_v, "CITATION.cff has no version")
33
+ self.assertIsNotNone(py_v, "pyproject.toml has no version")
34
+ self.assertEqual(cff_v.group(1).strip('"'), py_v.group(1),
35
+ "CITATION.cff version must equal pyproject version (bump both together)")
36
+
37
+ def test_no_broken_internal_doc_links(self):
38
+ # SH5: a Markdown link to a local file that 404s reads as abandonment for a tool that sells
39
+ # reviewability. The shipped docs must have zero broken internal links.
40
+ out = doc_link_check.check()
41
+ self.assertGreater(out["checked"], 0, "link checker found no local links to check — is it wired up?")
42
+ self.assertEqual(out["broken"], [], f"broken internal doc link(s): {out['broken']}")
43
+
44
+
45
+ if __name__ == "__main__":
46
+ unittest.main()
@@ -52,6 +52,29 @@ class TestEvalClaim(unittest.TestCase):
52
52
  self.assertTrue(verify_bundle(bundle).ok, f"{key}={bad}: bundle still signs/verifies")
53
53
  self.assertIsNone(decode_eval_claim(bundle), f"{key}={bad}: claim must NOT decode")
54
54
 
55
+ def test_decode_enforces_required_and_unknown_fields(self):
56
+ # F3 (v1.9.2): the exact key set is a VERIFY-path invariant, not only an emit-side one.
57
+ # emit_eval_receipt enforces _REQUIRED/_OPTIONAL, but a hand-signed claim (emit_bundle over a
58
+ # canonicalized dict) bypasses that path — previously such a claim decoded fine (the emit-vs-
59
+ # verify asymmetry class the project documents). The signature stays valid; only decode rejects.
60
+ from proofbundle.emit import emit_bundle
61
+ signer = generate_signer()
62
+ claim, _ = _claim(signer)
63
+ good = decode_eval_claim(emit_eval_receipt(claim, signer))
64
+ self.assertIsNotNone(good)
65
+ # (a) a claim missing a required field must NOT decode (issuer is checked separately, exclude it)
66
+ for drop in ("timestamp", "suite", "assurance_level", "n", "comparator"):
67
+ c = {k: v for k, v in good.items() if k != drop}
68
+ bundle = emit_bundle(canonicalize(c), signer)
69
+ self.assertTrue(verify_bundle(bundle).ok, f"drop {drop}: signature must still verify")
70
+ self.assertIsNone(decode_eval_claim(bundle), f"missing {drop}: claim must NOT decode")
71
+ # (b) a claim carrying an unknown field must NOT decode
72
+ c = dict(good)
73
+ c["totally_unknown_field"] = "x"
74
+ bundle = emit_bundle(canonicalize(c), signer)
75
+ self.assertTrue(verify_bundle(bundle).ok, "unknown field: signature must still verify")
76
+ self.assertIsNone(decode_eval_claim(bundle), "unknown field: claim must NOT decode")
77
+
55
78
  def test_decode_reads_path_once_no_toctou(self):
56
79
  # CRITICAL (release review): decode_eval_claim(path) must resolve the path to a dict EXACTLY ONCE and
57
80
  # verify + parse the SAME object. A second re-read is a TOCTOU (CWE-367) file-race window that could return
@@ -266,6 +266,22 @@ class TestBundleIntegration(unittest.TestCase):
266
266
  self.assertIn("NO issuer key", kb[0].detail)
267
267
  self.assertFalse(result.ok, "cnf-bound SD-JWT without an issuer key must fail the bundle")
268
268
 
269
+ def test_f4_expected_binding_without_kb_jwt_fails_closed(self):
270
+ # F4 (v1.9.2): supplying expected_aud/expected_nonce requests RFC 9901 §7.3 replay/audience
271
+ # binding. A bundle with NO verifiable KB-JWT (here: no sd_jwt_vc at all) carries nothing to
272
+ # bind to — verify must FAIL closed instead of silently returning OK (the downgrade trap: the
273
+ # verifier believes the presentation was bound to its aud/nonce when nothing bound it). A
274
+ # verifier that supplies no expected_* is unaffected (backward-compatible).
275
+ b = emit_bundle(b'{"x":1}', generate_signer()) # plain bundle, no sd_jwt_vc
276
+ self.assertTrue(verify_bundle(b).ok, "no expected_* → OK (backward-compatible)")
277
+ for kwargs in ({"expected_aud": "verifier.example"}, {"expected_nonce": "n-1"},
278
+ {"expected_aud": "v", "expected_nonce": "n"}):
279
+ res = verify_bundle(b, **kwargs)
280
+ self.assertFalse(res.ok, f"{kwargs}: no KB-JWT → must fail closed")
281
+ kb = [c for c in res.checks if c.name == "sd-jwt-key-binding"]
282
+ self.assertEqual(len(kb), 1, f"{kwargs}: must record exactly one refused binding check")
283
+ self.assertFalse(kb[0].ok)
284
+
269
285
  def test_bundle_no_cnf_no_issuer_key_still_backward_compatible(self):
270
286
  # The fix must NOT break plain SD-JWTs that carry no cnf and no KB — they verify as
271
287
  # structure-only exactly as before (no spurious key-binding verdict).
File without changes
File without changes