proofbundle 1.7.0__tar.gz → 1.8.0__tar.gz

{proofbundle-1.7.0/src/proofbundle.egg-info → proofbundle-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proofbundle
-Version: 1.7.0
+Version: 1.8.0
 Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
 Author: Konrad Gruszka
 License: MIT
@@ -84,7 +84,7 @@ file, no server, no network.**
 
 **At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
 verify` checks one self-contained `bundle.json` with three offline cryptographic
-checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
+checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 289 tests + a CI mutation gate.
 
 ## Contents
 

{proofbundle-1.7.0 → proofbundle-1.8.0}/README.md

@@ -36,7 +36,7 @@ file, no server, no network.**
 
 **At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
 verify` checks one self-contained `bundle.json` with three offline cryptographic
-checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
+checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 289 tests + a CI mutation gate.
 
 ## Contents
 

{proofbundle-1.7.0 → proofbundle-1.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "proofbundle"
-version = "1.7.0"
+version = "1.8.0"
 description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
 readme = "README.md"
 requires-python = ">=3.10"

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/__init__.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-__version__ = "1.7.0"
+__version__ = "1.8.0"
 
 __all__ = [
     "__version__",
@@ -36,6 +36,8 @@ __all__ = [
     "sample_opening",
     "verify_sample_opening",
     "audit_challenge",
+    "prereg_hash",
+    "verify_prereg",
     "VerificationResult",
     "Check",
     "ProofBundleError",
@@ -59,6 +61,8 @@ _LAZY = {
     "sample_opening": ".persample",
     "verify_sample_opening": ".persample",
     "audit_challenge": ".persample",
+    "prereg_hash": ".prereg",
+    "verify_prereg": ".prereg",
 }
 
 if TYPE_CHECKING:  # static analysers + IDEs see the real names/types; runtime stays lazy
@@ -70,6 +74,7 @@ if TYPE_CHECKING:  # static analysers + IDEs see the real names/types; runtime s
     from .hf_evals import receipt_token, verify_receipt_token
     from .persample import (audit_challenge, build_sample_tree, sample_opening,
                             verify_sample_opening)
+    from .prereg import prereg_hash, verify_prereg
     from .statuslist import verify_status_snapshot
     from .tlogproof import verify_tlog_proof
     from .merkle import verify_consistency, verify_inclusion

proofbundle-1.8.0/src/proofbundle/adapters/_provenance.py

@@ -0,0 +1,63 @@
+"""Shared provenance helpers for adapters (v1.8).
+
+The external review found that run-id and config-hash were missing from nearly every adapter and
+that the two flagship adapters took the timestamp from the caller rather than the eval log. These
+helpers close that: each adapter now records, where the framework exposes it, a stable RUN id, a
+CONFIG hash, and the LOG-NATIVE timestamp — so a receipt is traceable back to the exact run.
+
+Design notes (verified against framework source, 2026-07):
+  - No framework ships a canonical config hash, so we compute our own. Config is an in-memory
+    object re-serialized non-deterministically, so it MUST be canonicalized before hashing —
+    RFC 8785 JCS via the same `rfc8785` extra the emit path already needs. If that extra is
+    absent we fall back to a deterministic `json.dumps(sort_keys=True)` and LABEL the hash
+    algorithm accordingly, so a verifier is never misled about how the hash was formed.
+  - The hash is over the config's JSON, prefixed with a domain tag, hex sha256. It is provenance
+    metadata (traceability), NOT a security commitment — it is not salted and reveals structure;
+    it exists so two receipts from the same config are linkable and a changed config is visible.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Optional
+
+_CONFIG_DOMAIN = b"proofbundle/v1.8/config-hash\x00"
+
+
+def config_hash(config) -> Optional[str]:
+    """Return ``"<alg>:<hex>"`` over the canonical JSON of a config object, or None if it is
+    empty/None. ``<alg>`` is ``sha256-jcs`` when RFC 8785 is available, else ``sha256-sortkeys``
+    (both deterministic; the label tells a verifier which normalization produced the hex)."""
+    if config is None or config == {} or config == []:
+        return None
+    try:
+        import rfc8785  # noqa: PLC0415 — same optional dep as the emit path
+        canonical = rfc8785.dumps(config)
+        alg = "sha256-jcs"
+    except (ImportError, ValueError, TypeError):
+        # rfc8785 rejects non-JCS-able values (e.g. floats it deems unsafe); fall back to a
+        # deterministic stdlib serialization and label it so the difference is never hidden.
+        try:
+            canonical = json.dumps(config, sort_keys=True, separators=(",", ":"),
+                                   ensure_ascii=False).encode("utf-8")
+        except (TypeError, ValueError):
+            return None
+        alg = "sha256-sortkeys"
+    return f"{alg}:{hashlib.sha256(_CONFIG_DOMAIN + canonical).hexdigest()}"
+
+
+def add_provenance(provenance: dict, *, run_id=None, config=None, log_timestamp=None,
+                   config_hash_value: Optional[str] = None) -> dict:
+    """Merge the standard traceability fields into a provenance dict, skipping absent ones.
+
+    ``config_hash_value`` lets a caller pass a precomputed hash (e.g. over already-canonical
+    material) instead of a config object; otherwise ``config`` is hashed here."""
+    if run_id:
+        provenance["run_id"] = str(run_id)
+    if log_timestamp is not None:
+        provenance["run_timestamp"] = str(log_timestamp)
+    ch = config_hash_value if config_hash_value is not None else config_hash(config)
+    if ch:
+        provenance["config_hash"] = ch
+    return provenance

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/adapters/inspect_ai.py

@@ -90,6 +90,15 @@ def from_inspect_ai_log(path, metric: str, *, comparator: str, threshold: str, t
     if tv is not None:
         provenance["task_version"] = str(tv)
 
+    # v1.8 (external review): run-id + config-hash + LOG-NATIVE timestamp so a receipt traces back
+    # to the exact run. inspect_ai: eval.run_id (unique run id), eval.created (UTC datetime string),
+    # eval.task_args (the config material — no native config hash exists, so we compute one).
+    from ._provenance import add_provenance  # noqa: PLC0415
+    task_args = getattr(ev, "task_args", None)
+    add_provenance(provenance, run_id=getattr(ev, "run_id", None),
+                   config=task_args if isinstance(task_args, dict) else None,
+                   log_timestamp=getattr(ev, "created", None))
+
     return build_eval_claim(
         suite=suite, suite_version=str(getattr(ev, "task_version", "1")),
         metric=metric, comparator=comparator, threshold=threshold, score=_score_str(value),

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/adapters/lm_eval.py

@@ -71,6 +71,16 @@ def from_lm_eval_results(path, task: str, metric: str, *, comparator: str, thres
     if stderr is not None:
         provenance["stderr"] = repr(stderr) if not isinstance(stderr, str) else stderr
 
+    # v1.8 (external review): config-hash + LOG-NATIVE timestamp. lm-eval has no dedicated run-id;
+    # its `date` is a Unix float (distinct from the ISO filename stamp). `config` is the run config
+    # block (model/args/seeds/gen_kwargs); no native hash exists, so we compute one.
+    from ._provenance import add_provenance  # noqa: PLC0415
+    add_provenance(provenance, config=cfg if isinstance(cfg, dict) and cfg else None,
+                   log_timestamp=data.get("date"))
+    task_hashes = data.get("task_hashes", {})
+    if isinstance(task_hashes, dict) and task_hashes.get(task):
+        provenance["task_hash"] = str(task_hashes[task])   # lm-eval's native per-task sample hash
+
     return build_eval_claim(
         suite=task, suite_version=str(data.get("versions", {}).get(task, "lm-eval")),
         metric=metric, comparator=comparator, threshold=threshold, score=str(score), n=n,

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/adapters/promptfoo.py

@@ -124,6 +124,13 @@ def from_promptfoo_results(path, *, comparator: str, threshold: str, timestamp:
     if summary.get("timestamp"):
         provenance["run_timestamp"] = str(summary["timestamp"])
 
+    # v1.8 (external review): a uniform run_id key across adapters + a config-hash over the FULL
+    # resolved suite config (providers/prompts/tests/…), not just the tests-derived dataset id.
+    from ._provenance import add_provenance  # noqa: PLC0415
+    add_provenance(provenance, run_id=eval_id,
+                   config=config if isinstance(config, dict) and config else None,
+                   log_timestamp=metadata.get("evaluationCreatedAt"))
+
     return build_eval_claim(
         suite=suite, suite_version=f"promptfoo-summary-v{version}",
         metric="pass_rate", comparator=comparator, threshold=threshold, score=score, n=n,

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/cli.py

@@ -229,6 +229,37 @@ def _cmd_demo(args: argparse.Namespace) -> int:
     return run_demo(as_json=args.json)
 
 
+def _cmd_prereg(args: argparse.Namespace) -> int:
+    from .prereg import prereg_hash, verify_prereg  # noqa: PLC0415
+    try:
+        if args.check is not None:
+            from .evalclaim import decode_eval_claim  # noqa: PLC0415
+            # Release-review CRITICAL: --check MUST verify the receipt (Ed25519 + Merkle) BEFORE trusting its
+            # prereg_sha256 — the old load_bundle+manual-decode read an UNAUTHENTICATED claim, so a forged/unsigned
+            # bundle with a doctored prereg_sha256 got a false PASS (the exact anti-cherry-picking bypass this guards).
+            claim = decode_eval_claim(args.check)
+            if claim is None:
+                print("=> FAILED: not a valid, issuer-bound eval receipt", file=sys.stderr)
+                return 1
+            res = verify_prereg(args.protocol, claim)
+            if args.json:
+                print(json.dumps(res))
+            else:
+                print(f"[{'PASS' if res['ok'] else 'FAIL'}] prereg: {res['detail']}")
+            return 0 if res["ok"] else 1
+        h = prereg_hash(args.protocol)
+        if args.json:
+            print(json.dumps({"prereg_sha256": h}))
+        else:
+            print(h)
+            print("place this in the eval claim's prereg_sha256 BEFORE running the eval",
+                  file=sys.stderr)
+        return 0
+    except (ProofBundleError, OSError, ValueError, KeyError) as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="proofbundle",
@@ -317,6 +348,15 @@ def build_parser() -> argparse.ArgumentParser:
     demo.add_argument("--json", action="store_true", help="machine readable output")
     demo.set_defaults(func=_cmd_demo)
 
+    prereg = sub.add_parser(
+        "prereg",
+        help="hash an eval protocol file to commit to it BEFORE the run (--check verifies a receipt)")
+    prereg.add_argument("protocol", help="path to the protocol/plan file to hash")
+    prereg.add_argument("--check", metavar="RECEIPT",
+                        help="verify the protocol matches a receipt's prereg_sha256 instead of hashing")
+    prereg.add_argument("--json", action="store_true", help="machine readable output")
+    prereg.set_defaults(func=_cmd_prereg)
+
     return parser
 
 

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/evalclaim.py

@@ -274,6 +274,15 @@ def decode_eval_claim(bundle, *, expected_context: Optional[str] = None) -> Opti
         want = "ed25519:" + base64.b64encode(base64.b64decode(sig_pub_b64)).decode("ascii")
         if claim.get("issuer") != want:
             return None
+        # Verify-boundary schema invariants (release-review CRITICAL): emit_eval_receipt signs a hand-built claim
+        # WITHOUT build_eval_claim's checks, so a signed claim could carry an out-of-enum comparator or a
+        # non-decimal/non-finite threshold ("inf") — either silently collapses a downstream verdict check (e.g. the
+        # HF value-vs-verdict guard) into a tautology. Enforce them here, fail-closed, so every decoded claim is sane.
+        if claim.get("comparator") not in _COMPARATORS:
+            return None
+        _thr = claim.get("threshold")
+        if not (isinstance(_thr, str) and _DECIMAL_RE.match(_thr)):
+            return None
         samples = claim.get("samples")
         if samples is not None:
             if not isinstance(samples, dict) or set(samples) != {"root_b64", "n", "leaf_alg"}:
@@ -326,7 +335,7 @@ def check_freshness(claim: dict, max_age_seconds: Optional[int] = None, now=None
     """Replay check (v1.1): parse the claim's timestamp and report its age. A receipt carries a timestamp but
     verify never judged it — an old receipt could be replayed as new. Returns
     {"parsed": bool, "age_seconds": int|None, "fresh": bool|None, "reason": str}. ``fresh`` is None when no
-    ``max_age_seconds`` bound is given (age reported, not judged). 3.9-safe ISO parsing (normalizes a 'Z')."""
+    ``max_age_seconds`` bound is given (age reported, not judged). ISO parsing (normalizes a trailing Z)."""
     from datetime import datetime, timezone  # noqa: PLC0415
     ts = claim.get("timestamp")
     if not isinstance(ts, str):

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/hf_evals.py

@@ -85,7 +85,8 @@ def to_eval_results_entry(bundle: dict, *, dataset_id: str, task_id: str, value,
                           date: Optional[str] = None, source_url: Optional[str] = None,
                           source_name: Optional[str] = None, source_user: Optional[str] = None,
                           notes: Optional[str] = None, include_token: bool = True,
-                          require_verified: bool = True) -> dict:
+                          require_verified: bool = True,
+                          allow_value_mismatch: bool = False) -> dict:
     """Build one HF `.eval_results/*.yaml` entry for a receipt.
 
     ``dataset_id``/``task_id`` name the Hub benchmark (per its `eval.yaml`); ``value`` is the
@@ -96,6 +97,11 @@ def to_eval_results_entry(bundle: dict, *, dataset_id: str, task_id: str, value,
     is never generated from a broken receipt. ``include_token=True`` puts the ``pb1.`` token in
     ``verifyToken`` (schema-valid, proofbundle-verifiable; NOT the HF-internal badge token — HF's
     "verified" badge is HF's server-side decision, and this module makes no claim about it).
+
+    v1.8 (external review): if the bundle is an eval receipt whose claim discloses a ``score``,
+    the published ``value`` MUST match it (a Hub reader sees the value, not the token) — a
+    mismatch raises unless ``allow_value_mismatch=True``. This stops a receipt saying 0.60 from
+    being published next to a displayed 0.99.
     """
     if require_verified:
         result = verify_bundle(bundle)
@@ -118,6 +124,42 @@ def to_eval_results_entry(bundle: dict, *, dataset_id: str, task_id: str, value,
         raise BundleFormatError(
             "value must be a finite number — inf/-inf/nan cannot be represented in eval_results.yaml")
 
+    # v1.8 (external review): if the receipt is an eval claim, the published value must be
+    # CONSISTENT with the signed pass/fail verdict — a Hub reader sees the value, not the token,
+    # so publishing a value that contradicts the receipt is exactly the honesty gap to close.
+    # The claim minimizes data (it carries threshold/comparator/passed, not the exact score), so
+    # we check the strongest thing available: value <comparator> threshold must equal passed.
+    if not allow_value_mismatch:
+        from .evalclaim import EVAL_CLAIM_SCHEMA, decode_eval_claim  # noqa: PLC0415
+        claim = decode_eval_claim(bundle)
+        if claim is not None:
+            # decode_eval_claim now guarantees comparator ∈ the 4-value enum and a decimal (finite) threshold, so the
+            # lookup below is total and thr is finite — no "=="/"inf" tautology can silently no-op the check.
+            if {"threshold", "comparator", "passed"} <= set(claim):
+                thr = float(claim["threshold"])
+                cmp_ok = {">=": numeric >= thr, ">": numeric > thr,
+                          "<=": numeric <= thr, "<": numeric < thr}[claim["comparator"]]
+                if cmp_ok != bool(claim["passed"]):
+                    raise BundleFormatError(
+                        f"published value {numeric} is inconsistent with the receipt: the signed claim "
+                        f"says passed={claim['passed']} for {claim['comparator']} {claim['threshold']}, "
+                        f"but {numeric} {claim['comparator']} {claim['threshold']} is {cmp_ok} — "
+                        "pass allow_value_mismatch=True only if this is intentional")
+        else:
+            # decode failed. FAIL-CLOSED (release-review CRITICAL) if the payload IS an eval claim but did not decode
+            # (e.g. out-of-enum comparator / non-decimal threshold) — refusing to publish an unchecked value. A
+            # genuinely NON-eval bundle (different/absent schema) has no verdict to check → skip. The bundle's signature
+            # was already verified above (require_verified), so reading the raw payload's schema label is authentic.
+            try:
+                _raw = json.loads(base64.b64decode(bundle["payload_b64"]).decode("utf-8"))
+                _is_eval = isinstance(_raw, dict) and _raw.get("schema") == EVAL_CLAIM_SCHEMA
+            except (ValueError, TypeError, KeyError):
+                _is_eval = False
+            if _is_eval:
+                raise BundleFormatError(
+                    "cannot verify the published value against the receipt — the eval claim did not decode "
+                    "(invalid comparator/threshold?); pass allow_value_mismatch=True only if intentional")
+
     entry: dict = {"dataset": {"id": dataset_id, "task_id": task_id},
                    "value": numeric if isinstance(value, str) else value}
     if include_token:

proofbundle-1.8.0/src/proofbundle/prereg.py

@@ -0,0 +1,58 @@
+"""Pre-registration helper (v1.8) — commit to an eval protocol BEFORE the run.
+
+The single mitigation for best-of-many / cherry-picking that a receipt can carry: hash the
+protocol document (the plan — suite, seeds, decision rule, sampling policy) *before* running the
+eval, put that hash in the claim's ``prereg_sha256``, and sign the receipt. A verifier who is
+later handed the protocol file re-hashes it and checks it matches — so the plan could not have
+been written to fit the result. The signed receipt's own timestamp binds "this hash existed at
+receipt time" without any network dependency.
+
+Construction (verified against standards, 2026-07): commit = **sha256 over the RAW file bytes**.
+Document commitments hash raw bytes (git blob addressing, RFC 6962 leaf hashing, in-toto
+``gitBlob``/``sha256`` DigestSet all hash the artifact's own bytes) — NOT a re-normalized form.
+Canonicalization would only add a lossy transform the verifier must reproduce byte-for-byte; a
+trailing-newline or CRLF change breaking the match is tamper-evidence, not a bug. The claim field
+is a bare 64-hex ``prereg_sha256`` (matching the eval-claim schema).
+
+Out of scope, stated honestly: this proves the protocol was fixed relative to the receipt's
+signing time; it does NOT prove the run actually followed the protocol, nor timestamp the
+commitment against a third party's clock. An optional RFC 3161 TSA countersignature over the
+hash (e.g. FreeTSA) is the upgrade when the verifier does not trust the issuer's clock — that is
+a deployment choice, not built in here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+__all__ = ["prereg_hash", "verify_prereg"]
+
+
+def prereg_hash(protocol_path) -> str:
+    """Return the lowercase-hex sha256 over the RAW bytes of the protocol file — the value to
+    place in a claim's ``prereg_sha256`` BEFORE running the eval."""
+    data = Path(protocol_path).read_bytes()
+    return hashlib.sha256(data).hexdigest()
+
+
+def verify_prereg(protocol_path, claim: dict) -> dict:
+    """Check that ``claim['prereg_sha256']`` matches the sha256 of the protocol file.
+
+    Returns ``{ok, present, expected, actual, detail}``. ``present`` is False when the claim
+    carries no ``prereg_sha256`` (not pre-registered) — the caller decides whether that is
+    acceptable; ``ok`` is only True on a present-and-matching hash (fail-closed)."""
+    expected = claim.get("prereg_sha256") if isinstance(claim, dict) else None
+    result = {"ok": False, "present": expected is not None, "expected": expected,
+              "actual": None, "detail": ""}
+    if expected is None:
+        result["detail"] = "claim carries no prereg_sha256 (not pre-registered)"
+        return result
+    actual = prereg_hash(protocol_path)
+    result["actual"] = actual
+    if actual == expected:
+        result["ok"] = True
+        result["detail"] = "protocol file matches the pre-registered hash"
+    else:
+        result["detail"] = "protocol file does NOT match the pre-registered hash (plan changed?)"
+    return result

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle/tlogproof.py

@@ -97,7 +97,9 @@ def parse_tlog_proof(text: str) -> dict:
     if pos >= len(lines) or not lines[pos].startswith("index "):
         raise BundleFormatError("tlog-proof is missing the index line")
     index_s = lines[pos][len("index "):]
-    if not index_s.isdigit() or (index_s != "0" and index_s.startswith("0")):
+    # isascii() before isdigit() (release-review #7): str.isdigit() is True for Unicode digits (e.g. '²' U+00B2,
+    # Arabic-Indic), which int() would then reject or mis-parse — mirror checkpoint.py's ASCII-only guard.
+    if not (index_s.isascii() and index_s.isdigit()) or (index_s != "0" and index_s.startswith("0")):
         raise BundleFormatError("tlog-proof index must be ASCII decimal with no leading zeros")
     pos += 1
     proof = []

{proofbundle-1.7.0 → proofbundle-1.8.0/src/proofbundle.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proofbundle
-Version: 1.7.0
+Version: 1.8.0
 Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
 Author: Konrad Gruszka
 License: MIT
@@ -84,7 +84,7 @@ file, no server, no network.**
 
 **At a glance:** `proofbundle emit` signs and anchors a payload; `proofbundle
 verify` checks one self-contained `bundle.json` with three offline cryptographic
-checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 257 tests + a CI mutation gate.
+checks → `OK` or `FAILED`. No network, no daemon, no own crypto. 289 tests + a CI mutation gate.
 
 ## Contents
 

{proofbundle-1.7.0 → proofbundle-1.8.0}/src/proofbundle.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ src/proofbundle/intoto.py
 src/proofbundle/kbjwt.py
 src/proofbundle/merkle.py
 src/proofbundle/persample.py
+src/proofbundle/prereg.py
 src/proofbundle/py.typed
 src/proofbundle/pytest_plugin.py
 src/proofbundle/sdjwt.py
@@ -33,6 +34,7 @@ src/proofbundle.egg-info/entry_points.txt
 src/proofbundle.egg-info/requires.txt
 src/proofbundle.egg-info/top_level.txt
 src/proofbundle/adapters/__init__.py
+src/proofbundle/adapters/_provenance.py
 src/proofbundle/adapters/eee.py
 src/proofbundle/adapters/inspect_ai.py
 src/proofbundle/adapters/lm_eval.py
@@ -53,6 +55,7 @@ tests/test_emit.py
 tests/test_eval_claim_schema.py
 tests/test_evalclaim.py
 tests/test_examples.py
+tests/test_fuzz_parsers.py
 tests/test_hf_evals.py
 tests/test_inspect_hook.py
 tests/test_intoto.py
@@ -61,7 +64,9 @@ tests/test_kbjwt.py
 tests/test_merkle.py
 tests/test_merkle_property.py
 tests/test_persample.py
+tests/test_prereg.py
 tests/test_promptfoo.py
+tests/test_provenance.py
 tests/test_pytest_plugin.py
 tests/test_rekor_interop.py
 tests/test_rfc6962_external_vectors.py

{proofbundle-1.7.0 → proofbundle-1.8.0}/tests/test_evalclaim.py

@@ -39,6 +39,19 @@ class TestEvalClaim(unittest.TestCase):
         self.assertEqual(decoded["suite"], "safety-refusal")
         self.assertTrue(decoded["passed"])
 
+    def test_decode_rejects_bad_comparator_and_threshold(self):
+        # release-review CRITICAL: emit_eval_receipt signs a hand-built claim WITHOUT build_eval_claim's checks,
+        # so decode_eval_claim must enforce comparator-enum + decimal-threshold at the verify boundary — else a
+        # downstream value-consistency check silently no-ops on comparator "==" / non-finite threshold "inf".
+        signer = generate_signer()
+        for key, bad in (("comparator", "=="), ("comparator", "~="),
+                         ("threshold", "inf"), ("threshold", "nan"), ("threshold", "1e5")):
+            claim, _ = _claim(signer)
+            claim[key] = bad
+            bundle = emit_eval_receipt(claim, signer)
+            self.assertTrue(verify_bundle(bundle).ok, f"{key}={bad}: bundle still signs/verifies")
+            self.assertIsNone(decode_eval_claim(bundle), f"{key}={bad}: claim must NOT decode")
+
     def test_decode_reads_path_once_no_toctou(self):
         # CRITICAL (release review): decode_eval_claim(path) must resolve the path to a dict EXACTLY ONCE and
         # verify + parse the SAME object. A second re-read is a TOCTOU (CWE-367) file-race window that could return

proofbundle-1.8.0/tests/test_fuzz_parsers.py

@@ -0,0 +1,88 @@
+"""Property-based fuzzing of the text/JWT parsers (v1.8).
+
+The invariant for every attacker-controlled parser: on ANY input it returns a value or raises a
+proofbundle error (BundleFormatError / ProofBundleError / ValueError) — NEVER an uncaught crash
+(AttributeError, IndexError, KeyError, TypeError, UnicodeError, recursion, …) and never a hang.
+This is the "never a raw traceback" contract, checked adversarially with Hypothesis rather than
+by hand-picked cases. Hypothesis is the lowest-friction sound fuzzer for pure-Python parsers
+(no native toolchain); an Atheris coverage-guided driver over the same bodies can be added under
+fuzz/ for Linux CI if deeper coverage is ever wanted. hypothesis is a dev dependency only — this
+module no-ops when it is absent (same pattern as test_merkle_property.py)."""
+from __future__ import annotations
+
+import unittest
+
+try:
+    from hypothesis import given, settings
+    from hypothesis import strategies as st
+except ImportError:  # pragma: no cover - dev-only dependency
+    given = None
+
+from proofbundle.errors import ProofBundleError
+from proofbundle.tlogproof import parse_tlog_proof, verify_tlog_proof
+from proofbundle.checkpoint import verify_checkpoint, verify_cosignature
+from proofbundle.statuslist import verify_status_snapshot
+from proofbundle.kbjwt import split_key_binding, verify_key_binding
+from proofbundle.sdjwt import verify_sd_jwt
+
+_ALLOWED = (ProofBundleError, ValueError)   # the documented "malformed input" surface
+
+
+def _must_not_crash(fn, *args, **kwargs):
+    try:
+        fn(*args, **kwargs)
+    except _ALLOWED:
+        pass   # documented malformed-input path — fine
+    # any other exception propagates and fails the test (the contract violation we hunt)
+
+
+if given is not None:
+    _texts = st.text(alphabet=st.characters(min_codepoint=1, max_codepoint=0x2FFF), max_size=400)
+
+    class TestParserRobustness(unittest.TestCase):
+        @settings(max_examples=300, deadline=None)
+        @given(_texts)
+        def test_parse_tlog_proof_never_crashes(self, s):
+            _must_not_crash(parse_tlog_proof, s)
+
+        @settings(max_examples=200, deadline=None)
+        @given(_texts, _texts)
+        def test_verify_tlog_proof_never_crashes(self, proof, leaf):
+            _must_not_crash(verify_tlog_proof, proof, leaf.encode("utf-8", "surrogatepass"),
+                            "log+00000000+" + "A" * 44)
+
+        @settings(max_examples=300, deadline=None)
+        @given(_texts, _texts)
+        def test_verify_checkpoint_never_crashes(self, note, vkey):
+            _must_not_crash(verify_checkpoint, note, vkey)
+
+        @settings(max_examples=300, deadline=None)
+        @given(_texts, _texts)
+        def test_verify_cosignature_never_crashes(self, note, vkey):
+            _must_not_crash(verify_cosignature, note, vkey)
+
+        @settings(max_examples=300, deadline=None)
+        @given(_texts)
+        def test_split_key_binding_never_crashes(self, compact):
+            sd, kb = split_key_binding(compact)          # total by contract → returns a tuple
+            self.assertIsInstance(sd, str)
+
+        @settings(max_examples=300, deadline=None)
+        @given(_texts)
+        def test_verify_key_binding_never_crashes(self, compact):
+            _must_not_crash(verify_key_binding, compact)
+
+        @settings(max_examples=300, deadline=None)
+        @given(_texts)
+        def test_verify_sd_jwt_never_crashes(self, compact):
+            _must_not_crash(verify_sd_jwt, compact)
+
+        @settings(max_examples=200, deadline=None)
+        @given(_texts)
+        def test_verify_status_snapshot_never_crashes(self, token):
+            _must_not_crash(verify_status_snapshot, token, expected_uri="u", index=0,
+                            issuer_pubkey=b"\x00" * 32)
+
+
+if __name__ == "__main__":
+    unittest.main()

{proofbundle-1.7.0 → proofbundle-1.8.0}/tests/test_hf_evals.py

@@ -150,3 +150,38 @@ class TestEvalResultsEntry(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestValueConsistency(unittest.TestCase):
+    """v1.8: a published value that contradicts the signed pass/fail verdict is refused."""
+
+    def _receipt(self, threshold, comparator, score):
+        from proofbundle import generate_signer
+        from proofbundle.evalclaim import build_eval_claim, emit_eval_receipt
+        claim, _ = build_eval_claim(
+            suite="s", suite_version="1", metric="acc", comparator=comparator,
+            threshold=threshold, score=score, n=10, model_id="m", dataset_id="d",
+            issuer="", timestamp="2026-07-02T00:00:00Z")
+        return emit_eval_receipt(claim, generate_signer())
+
+    def test_consistent_value_ok(self):
+        r = self._receipt("0.80", ">=", "0.91")   # passed=True
+        entry = to_eval_results_entry(r, dataset_id="d/x", task_id="t", value=0.91)
+        self.assertEqual(entry["value"], 0.91)
+
+    def test_inconsistent_value_refused(self):
+        r = self._receipt("0.80", ">=", "0.60")   # passed=False (0.60 < 0.80)
+        with self.assertRaises(BundleFormatError) as ctx:
+            to_eval_results_entry(r, dataset_id="d/x", task_id="t", value=0.99)  # 0.99>=0.80 → True, but claim says False
+        self.assertIn("inconsistent", str(ctx.exception))
+
+    def test_override_allows_mismatch(self):
+        r = self._receipt("0.80", ">=", "0.60")
+        entry = to_eval_results_entry(r, dataset_id="d/x", task_id="t", value=0.99,
+                                      allow_value_mismatch=True)
+        self.assertEqual(entry["value"], 0.99)
+
+    def test_non_eval_bundle_skips_check(self):
+        # a plain emit_bundle (not an eval receipt) has no claim → no cross-check, value accepted
+        entry = to_eval_results_entry(_bundle(), dataset_id="d/x", task_id="t", value=0.5)
+        self.assertEqual(entry["value"], 0.5)

proofbundle-1.8.0/tests/test_prereg.py

@@ -0,0 +1,122 @@
+"""v1.8 pre-registration helper: commit to a protocol before the run, verify after."""
+import hashlib
+import json
+import os
+import tempfile
+import unittest
+
+from proofbundle.cli import main
+from proofbundle.emit import generate_signer
+from proofbundle.evalclaim import build_eval_claim, emit_eval_receipt, issuer_fingerprint
+from proofbundle.prereg import prereg_hash, verify_prereg
+
+
+class TestPrereg(unittest.TestCase):
+    def _file(self, content: bytes) -> str:
+        h = tempfile.NamedTemporaryFile("wb", delete=False)
+        h.write(content)
+        h.close()
+        return h.name
+
+    def test_hash_is_sha256_of_raw_bytes(self):
+        content = b"protocol: fixed seeds 1..5\ndecision: acc >= 0.8\n"
+        path = self._file(content)
+        try:
+            self.assertEqual(prereg_hash(path), hashlib.sha256(content).hexdigest())
+        finally:
+            os.unlink(path)
+
+    def test_verify_match(self):
+        content = b"the plan"
+        path = self._file(content)
+        try:
+            claim = {"prereg_sha256": hashlib.sha256(content).hexdigest()}
+            res = verify_prereg(path, claim)
+            self.assertTrue(res["ok"])
+            self.assertTrue(res["present"])
+        finally:
+            os.unlink(path)
+
+    def test_verify_mismatch_is_caught(self):
+        path = self._file(b"the ACTUAL plan")
+        try:
+            claim = {"prereg_sha256": hashlib.sha256(b"a DIFFERENT plan").hexdigest()}
+            res = verify_prereg(path, claim)
+            self.assertFalse(res["ok"])
+            self.assertIn("does NOT match", res["detail"])
+        finally:
+            os.unlink(path)
+
+    def test_not_preregistered_reports_absent(self):
+        path = self._file(b"x")
+        try:
+            res = verify_prereg(path, {})           # no prereg_sha256
+            self.assertFalse(res["ok"])
+            self.assertFalse(res["present"])
+        finally:
+            os.unlink(path)
+
+    def test_trailing_byte_change_breaks_match(self):
+        # tamper-evidence: a single appended newline changes the commitment (by design).
+        path = self._file(b"plan\n")
+        try:
+            claim = {"prereg_sha256": hashlib.sha256(b"plan").hexdigest()}   # committed without \n
+            self.assertFalse(verify_prereg(path, claim)["ok"])
+        finally:
+            os.unlink(path)
+
+    def test_cli_roundtrip(self):
+        import json
+        from proofbundle.cli import main
+        from proofbundle import generate_signer
+        from proofbundle.evalclaim import build_eval_claim, emit_eval_receipt
+        import contextlib
+        import io
+        proto = self._file(b"suite=mmlu; seeds=1..5; rule=acc>=0.8")
+        try:
+            h = prereg_hash(proto)
+            claim, _ = build_eval_claim(
+                suite="s", suite_version="1", metric="acc", comparator=">=", threshold="0.8",
+                score="0.9", n=10, model_id="m", dataset_id="d", issuer="",
+                timestamp="2026-07-02T00:00:00Z", prereg_sha256=h)
+            receipt = emit_eval_receipt(claim, generate_signer())
+            rp = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False)
+            json.dump(receipt, rp)
+            rp.close()
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(main(["prereg", proto, "--check", rp.name]), 0)
+                # a different protocol fails
+                other = self._file(b"a different plan")
+                self.assertEqual(main(["prereg", other, "--check", rp.name]), 1)
+            os.unlink(rp.name)
+            os.unlink(other)
+        finally:
+            os.unlink(proto)
+
+
+class TestPreregCheckVerifies(unittest.TestCase):
+    def test_check_rejects_forged_unsigned_bundle(self):
+        # release-review CRITICAL: prereg --check MUST verify the receipt's signature before trusting its
+        # prereg_sha256 — a forged/unsigned bundle with a doctored prereg_sha256 must NOT get a PASS.
+        with tempfile.TemporaryDirectory() as d:
+            proto = os.path.join(d, "protocol.md")
+            with open(proto, "wb") as f:
+                f.write(b"pre-registered analysis plan v1")
+            h = prereg_hash(proto)
+            signer = generate_signer()
+            claim, _ = build_eval_claim(
+                suite="s", suite_version="v1", metric="m", comparator=">=", threshold="0.80",
+                score="0.92", n=10, model_id="a/b", dataset_id="c/d",
+                issuer=issuer_fingerprint(signer), timestamp="2026-07-01T12:00:00Z",
+                model_salt=b"0" * 16, dataset_salt=b"1" * 16, prereg_sha256=h)
+            bundle = emit_eval_receipt(claim, signer)
+            bundle["signature"]["signature_b64"] = "AA==" * 22    # corrupt the Ed25519 signature
+            fp = os.path.join(d, "forged.json")
+            with open(fp, "w", encoding="utf-8") as f:
+                json.dump(bundle, f)
+            rc = main(["prereg", proto, "--check", fp])
+            self.assertNotEqual(rc, 0, "a forged/unsigned receipt must FAIL prereg --check")
+
+
+if __name__ == "__main__":
+    unittest.main()

proofbundle-1.8.0/tests/test_provenance.py

@@ -0,0 +1,77 @@
+"""v1.8 provenance hardening: config-hash + run-id + log-native timestamp in adapters."""
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from proofbundle.adapters._provenance import add_provenance, config_hash
+from proofbundle.adapters import from_lm_eval_results, from_promptfoo_results
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+class TestConfigHash(unittest.TestCase):
+    def test_deterministic_and_labeled(self):
+        a = config_hash({"b": 1, "a": 2})
+        b = config_hash({"a": 2, "b": 1})            # key order must not matter
+        self.assertEqual(a, b)
+        self.assertTrue(a.startswith("sha256-jcs:") or a.startswith("sha256-sortkeys:"))
+        self.assertEqual(len(a.split(":")[1]), 64)
+
+    def test_empty_is_none(self):
+        self.assertIsNone(config_hash(None))
+        self.assertIsNone(config_hash({}))
+        self.assertIsNone(config_hash([]))
+
+    def test_change_changes_hash(self):
+        self.assertNotEqual(config_hash({"seed": 1}), config_hash({"seed": 2}))
+
+    def test_add_provenance_skips_absent(self):
+        prov = {}
+        add_provenance(prov, run_id=None, config=None, log_timestamp=None)
+        self.assertEqual(prov, {})
+        add_provenance(prov, run_id="r1", config={"x": 1}, log_timestamp=123)
+        self.assertEqual(prov["run_id"], "r1")
+        self.assertEqual(prov["run_timestamp"], "123")
+        self.assertIn("config_hash", prov)
+
+
+class TestPromptfooProvenance(unittest.TestCase):
+    def test_run_id_and_config_hash_present(self):
+        claim, _ = from_promptfoo_results(
+            FIXTURES / "promptfoo_results_v3.json", comparator=">=", threshold="0.5",
+            timestamp="2026-07-02T00:00:00Z")
+        prov = claim["provenance"]
+        self.assertEqual(prov["run_id"], "eval-Xa3-2026-07-02T14:03:11")
+        self.assertIn("config_hash", prov)
+        self.assertIn("run_timestamp", prov)
+
+
+class TestLmEvalProvenance(unittest.TestCase):
+    def _fixture_with_config(self):
+        data = json.loads((FIXTURES / "lm_eval_arc_easy_real.json").read_text())
+        return data
+
+    def test_config_hash_and_timestamp(self):
+        data = self._fixture_with_config()
+        data.setdefault("config", {"model": "hf", "seed": 1234})
+        data["date"] = 1780000000.5
+        handle = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False)
+        json.dump(data, handle)
+        handle.close()
+        try:
+            task = next(iter(data["results"]))
+            metric = next(k.split(",")[0] for k in data["results"][task] if "," in k)
+            claim, _ = from_lm_eval_results(handle.name, task=task, metric=metric,
+                                            comparator=">=", threshold="0.1",
+                                            timestamp="2026-07-02T00:00:00Z")
+        finally:
+            os.unlink(handle.name)
+        prov = claim["provenance"]
+        self.assertIn("config_hash", prov)
+        self.assertEqual(prov["run_timestamp"], "1780000000.5")   # log-native, not caller ts
+
+
+if __name__ == "__main__":
+    unittest.main()