proofbundle 3.6.3__tar.gz → 3.7.0__tar.gz

{proofbundle-3.6.3/src/proofbundle.egg-info → proofbundle-3.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proofbundle
-Version: 3.6.3
+Version: 3.7.0
 Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
 Author: Konrad Gruszka
 License: MIT

{proofbundle-3.6.3 → proofbundle-3.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "proofbundle"
-version = "3.6.3"
+version = "3.7.0"
 description = "Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT."
 readme = "README.md"
 requires-python = ">=3.10"

{proofbundle-3.6.3 → proofbundle-3.7.0}/src/proofbundle/__init__.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-__version__ = "3.6.3"
+__version__ = "3.7.0"
 
 # The `proofbundle/v0.1` normative spec revision this build implements — kept in sync with the
 # `Revision:` line at the top of SPEC.md by tests/test_docs_truth.py (WP-B1, closes #28). Bump

{proofbundle-3.6.3 → proofbundle-3.7.0}/src/proofbundle/adapters/lm_eval.py

@@ -56,12 +56,20 @@ def from_lm_eval_results(path, task: str, metric: str, *, comparator: str, thres
 
     n_samples = data.get("n-samples", {}).get(task, {})
     n = int(n_samples.get("effective") or n_samples.get("original") or res.get("sample_len") or 0)
+    effective_samples = int(n_samples["effective"]) if n_samples.get("effective") is not None else n
+    original_samples = (int(n_samples["original"])
+                        if n_samples.get("original") is not None else effective_samples)
+    if effective_samples < 0 or original_samples < 0:
+        raise ValueError("n-samples effective and original counts must be non-negative")
+    skipped_samples = max(original_samples - effective_samples, 0)
     cfg = data.get("config", {})
     model_id = str(cfg.get("model_name") or cfg.get("model") or "unknown")
     if cfg.get("model_args"):
         model_id = f"{model_id}::{cfg['model_args']}"   # include args so the commitment pins the exact model
 
-    provenance = {"harness": "lm-evaluation-harness", "matched_metric_key": matched}
+    provenance = {"harness": "lm-evaluation-harness", "matched_metric_key": matched,
+                  "effective_samples": effective_samples, "original_samples": original_samples,
+                  "skipped_samples": skipped_samples}
     if data.get("git_hash"):
         provenance["git_hash"] = str(data["git_hash"])
     if data.get("versions", {}).get(task) is not None:

{proofbundle-3.6.3 → proofbundle-3.7.0/src/proofbundle.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proofbundle
-Version: 3.6.3
+Version: 3.7.0
 Summary: Emit and verify portable cryptographic evidence bundles, offline: Ed25519 + RFC 6962 Merkle + optional SD-JWT.
 Author: Konrad Gruszka
 License: MIT

{proofbundle-3.6.3 → proofbundle-3.7.0}/tests/test_adapters.py

@@ -1,5 +1,7 @@
 """Adapters map real exported eval JSON to a valid claim (file-based, no framework import)."""
 from copy import deepcopy
+import json
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -21,6 +23,42 @@ class TestAdapters(unittest.TestCase):
         self.assertIn("git_hash", claim["provenance"])         # provenance captured
         self.assertEqual(claim["provenance"]["n_shot"], "0")
         self.assertIn("stderr", claim["provenance"])           # sibling stderr, not nested
+        self.assertEqual(claim["n"], 2)
+        self.assertEqual(claim["provenance"]["effective_samples"], 2)
+        self.assertEqual(claim["provenance"]["original_samples"], 2376)
+        self.assertEqual(claim["provenance"]["skipped_samples"], 2374)
+
+    def test_lm_eval_effective_above_original_clamps_skipped_to_zero(self):
+        data = json.loads((FX / "lm_eval_arc_easy_real.json").read_text())
+        clamped = deepcopy(data)
+        clamped["n-samples"]["arc_easy"]["original"] = 1     # effective (2) > original (1)
+        with tempfile.TemporaryDirectory() as raw:
+            p = Path(raw) / "clamped.json"
+            p.write_text(json.dumps(clamped))
+            claim, _ = from_lm_eval_results(p, task="arc_easy", metric="acc", comparator=">=",
+                                            threshold="0.30", timestamp=TS,
+                                            model_salt=b"0" * 16, dataset_salt=b"1" * 16)
+        self.assertEqual(claim["provenance"]["skipped_samples"], 0)   # clamp branch, raw counts visible
+        self.assertEqual(claim["provenance"]["effective_samples"], 2)
+        self.assertEqual(claim["provenance"]["original_samples"], 1)
+
+    def test_lm_eval_sample_drop_count_changes_signed_provenance(self):
+        data = json.loads((FX / "lm_eval_arc_easy_real.json").read_text())
+        changed = deepcopy(data)
+        changed["n-samples"]["arc_easy"]["original"] = 3
+        common = dict(task="arc_easy", metric="acc", comparator=">=", threshold="0.30",
+                      timestamp=TS, model_salt=b"0" * 16, dataset_salt=b"1" * 16)
+        with tempfile.TemporaryDirectory() as raw:
+            base_path = Path(raw) / "base.json"
+            changed_path = Path(raw) / "changed.json"
+            base_path.write_text(json.dumps(data))
+            changed_path.write_text(json.dumps(changed))
+            base_claim, _ = from_lm_eval_results(base_path, **common)
+            changed_claim, _ = from_lm_eval_results(changed_path, **common)
+        self.assertEqual(base_claim["n"], changed_claim["n"])
+        self.assertNotEqual(base_claim["provenance"], changed_claim["provenance"])
+        self.assertEqual(base_claim["provenance"]["skipped_samples"], 2374)
+        self.assertEqual(changed_claim["provenance"]["skipped_samples"], 1)
 
     def test_lm_eval_missing_metric_lists_available(self):
         with self.assertRaises(ValueError):