falsify-inspect 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "pip"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
7
+ day: "monday"
8
+ time: "06:00"
9
+ timezone: "Europe/Istanbul"
10
+ open-pull-requests-limit: 5
11
+ labels:
12
+ - "dependencies"
13
+
14
+ - package-ecosystem: "github-actions"
15
+ directory: "/"
16
+ schedule:
17
+ interval: "weekly"
18
+ day: "monday"
19
+ labels:
20
+ - "dependencies"
21
+ - "github-actions"
@@ -0,0 +1,66 @@
1
+ name: Publish to PyPI
2
+
3
+ # Triggered on a tag push (e.g. v0.1.0). Builds the dist and publishes
4
+ # via PyPI Trusted Publishers (OIDC). No API token required.
5
+ #
6
+ # Initial setup (one-time, on PyPI side):
7
+ # 1. Reserve the project name by creating a "pending publisher" at
8
+ # https://pypi.org/manage/account/publishing/ with:
9
+ # - Project name: falsify-inspect
10
+ # - Owner: studio-11-co
11
+ # - Repository: falsify-inspect
12
+ # - Workflow name: publish.yml
13
+ # - Environment: pypi (must match the `environment:` below)
14
+ # 2. After the first successful publish, the project converts to a
15
+ # regular trusted publisher.
16
+ # Docs: https://docs.pypi.org/trusted-publishers/
17
+
18
+ on:
19
+ push:
20
+ tags:
21
+ - "v*"
22
+ workflow_dispatch:
23
+
24
+ permissions:
25
+ contents: read
26
+ id-token: write # required for OIDC trust to PyPI
27
+
28
+ jobs:
29
+ build:
30
+ runs-on: ubuntu-latest
31
+ steps:
32
+ - uses: actions/checkout@v4
33
+
34
+ - uses: actions/setup-python@v5
35
+ with:
36
+ python-version: "3.12"
37
+
38
+ - name: Install build
39
+ run: python -m pip install --upgrade pip build
40
+
41
+ - name: Build sdist + wheel
42
+ run: python -m build
43
+
44
+ - name: Upload dist artifact
45
+ uses: actions/upload-artifact@v4
46
+ with:
47
+ name: dist
48
+ path: dist/
49
+
50
+ publish:
51
+ needs: build
52
+ runs-on: ubuntu-latest
53
+ environment:
54
+ name: pypi
55
+ url: https://pypi.org/p/falsify-inspect
56
+ permissions:
57
+ id-token: write
58
+ steps:
59
+ - name: Download dist artifact
60
+ uses: actions/download-artifact@v4
61
+ with:
62
+ name: dist
63
+ path: dist/
64
+
65
+ - name: Publish to PyPI
66
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,46 @@
1
+ name: OpenSSF Scorecard
2
+
3
+ on:
4
+ branch_protection_rule:
5
+ schedule:
6
+ - cron: "0 6 * * 1" # weekly, Monday 06:00 UTC
7
+ push:
8
+ branches: [main]
9
+
10
+ # Required permissions for the OpenSSF Scorecard action
11
+ permissions: read-all
12
+
13
+ jobs:
14
+ analysis:
15
+ name: Scorecard analysis
16
+ runs-on: ubuntu-latest
17
+ permissions:
18
+ security-events: write # for SARIF upload
19
+ id-token: write # for keyless signing of results
20
+ contents: read
21
+
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+ with:
25
+ persist-credentials: false
26
+
27
+ - name: Run analysis
28
+ uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
29
+ with:
30
+ results_file: results.sarif
31
+ results_format: sarif
32
+ # publish_results pushes a public scorecard.dev page for the repo.
33
+ # Requires the repo to be public; safe to leave true.
34
+ publish_results: true
35
+
36
+ - name: Upload artifact
37
+ uses: actions/upload-artifact@v4
38
+ with:
39
+ name: SARIF results
40
+ path: results.sarif
41
+ retention-days: 30
42
+
43
+ - name: Upload SARIF to code-scanning
44
+ uses: github/codeql-action/upload-sarif@v3
45
+ with:
46
+ sarif_file: results.sarif
@@ -0,0 +1,34 @@
1
+ name: tests
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ matrix:
14
+ python: ["3.11", "3.12", "3.13"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python }}
20
+ - name: Install
21
+ run: |
22
+ python -m pip install --upgrade pip
23
+ pip install -e ".[dev]"
24
+ - name: Run tests
25
+ run: pytest tests/ -v --cov=falsify_inspect --cov-report=term-missing
26
+ - name: CLI smoke test
27
+ run: |
28
+ h=$(falsify-inspect lock \
29
+ --metric refusal_rate --threshold 0.95 --threshold-direction ">=" \
30
+ --dataset harmbench-v1 --dataset-hash sha256:abc \
31
+ --model-version "claude-3.5-sonnet@2025-10-01" \
32
+ --sample-size 500 --seed 42 --task harmbench)
33
+ echo "hash: $h"
34
+ [[ "$h" =~ ^sha256: ]] || (echo "lock did not produce sha256: prefix"; exit 1)
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.pyc
3
+ *.pyo
4
+ *.egg-info/
5
+ build/
6
+ dist/
7
+ .venv/
8
+ .pytest_cache/
9
+ .coverage
10
+ htmlcov/
11
+ .DS_Store
12
+ *.log
@@ -0,0 +1,20 @@
1
+ # Changelog
2
+
3
+ All notable changes to `falsify-inspect` are documented here.
4
+
5
+ ## [0.1.0] — 2026-05-08
6
+
7
+ Initial public release.
8
+
9
+ ### Added
10
+ - `preregister()` — generate a PRML manifest hash before an Inspect AI eval
11
+ - `verify_eval_log()` — re-derive the hash from an eval log + caller-supplied threshold and check
12
+ - `extract_manifest_from_log()` — pull PRML-relevant metadata out of `inspect_ai` log JSON
13
+ - `falsify-inspect lock` CLI — pre-register and emit hash
14
+ - `falsify-inspect verify` CLI — verify a log against a hash, exit 0/3/10
15
+ - 8 unit tests covering deterministic hashing, threshold validation, log extraction, pass/fail/tamper paths
16
+
17
+ ### Notes
18
+ - Compatible with PRML v0.1 spec
19
+ - Inspect AI itself is an *optional* dependency — extraction works on the log JSON shape, not the live API
20
+ - Patent non-assertion grant per PRML v0.1 spec appendix
@@ -0,0 +1,53 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it as below."
3
+ title: "falsify-inspect: PRML pre-registration adapter for Inspect AI eval logs"
4
+ abstract: >-
5
+ A Python package that wraps the Inspect AI evaluation framework with
6
+ Pre-Registered ML Manifest (PRML) v0.1 hashing, allowing eval claim
7
+ thresholds to be cryptographically committed before a run and
8
+ verified against the eval log afterwards.
9
+ authors:
10
+ - family-names: "Öztürk"
11
+ given-names: "Cüneyt"
12
+ affiliation: "Studio 11 Turkey Ltd. Şti."
13
+ email: "cuneyt@studio-11.co"
14
+ type: software
15
+ license: MIT
16
+ repository-code: "https://github.com/studio-11-co/falsify-inspect"
17
+ url: "https://falsify.dev"
18
+ version: "0.1.0"
19
+ date-released: 2026-05-08
20
+ keywords:
21
+ - "PRML"
22
+ - "pre-registration"
23
+ - "Inspect AI"
24
+ - "ML evaluation"
25
+ - "AI safety"
26
+ - "reproducibility"
27
+ identifiers:
28
+ - type: url
29
+ value: "https://spec.falsify.dev/v0.1"
30
+ description: "PRML v0.1 specification"
31
+ preferred-citation:
32
+ type: software
33
+ title: "falsify-inspect"
34
+ authors:
35
+ - family-names: "Öztürk"
36
+ given-names: "Cüneyt"
37
+ year: 2026
38
+ url: "https://github.com/studio-11-co/falsify-inspect"
39
+ version: "0.1.0"
40
+ references:
41
+ - type: software
42
+ title: "falsify — PRML reference implementation"
43
+ authors:
44
+ - family-names: "Öztürk"
45
+ given-names: "Cüneyt"
46
+ repository-code: "https://github.com/studio-11-co/falsify"
47
+ - type: article
48
+ title: "Pre-Registered ML Manifests: A Cryptographic Primitive for Verifiable Evaluation Claims"
49
+ authors:
50
+ - family-names: "Öztürk"
51
+ given-names: "Cüneyt"
52
+ year: 2026
53
+ url: "https://spec.falsify.dev/v0.1"
@@ -0,0 +1,22 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Studio 11 Turkey Ltd. Şti.
4
+ Author: Cüneyt Öztürk <cuneyt@studio-11.co>
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in all
14
+ copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ SOFTWARE.
@@ -0,0 +1,140 @@
1
+ Metadata-Version: 2.4
2
+ Name: falsify-inspect
3
+ Version: 0.1.0
4
+ Summary: PRML pre-registration integration for Inspect AI eval logs
5
+ Project-URL: Homepage, https://falsify.dev
6
+ Project-URL: Spec, https://spec.falsify.dev/v0.1
7
+ Project-URL: Repository, https://github.com/studio-11-co/falsify-inspect
8
+ Project-URL: Issues, https://github.com/studio-11-co/falsify-inspect/issues
9
+ Author: Studio 11 Turkey Ltd. Şti.
10
+ Author-email: Cüneyt Öztürk <cuneyt@studio-11.co>
11
+ License: MIT
12
+ License-File: LICENSE
13
+ Keywords: ai-safety,evaluation,inspect-ai,ml,pre-registration,prml,reproducibility
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Classifier: Topic :: Software Development :: Testing
22
+ Requires-Python: >=3.11
23
+ Requires-Dist: falsify>=0.1.4
24
+ Requires-Dist: pyyaml>=6.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
27
+ Requires-Dist: pytest>=7.0; extra == 'dev'
28
+ Provides-Extra: inspect
29
+ Requires-Dist: inspect-ai>=0.3.0; extra == 'inspect'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # falsify-inspect
33
+
34
+ PRML pre-registration for [Inspect AI](https://github.com/UKGovernmentBEIS/inspect_ai) eval logs.
35
+
36
+ [![PyPI](https://img.shields.io/pypi/v/falsify-inspect.svg)](https://pypi.org/project/falsify-inspect/)
37
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
38
+ [![PRML v0.1](https://img.shields.io/badge/PRML-v0.1-39D98A.svg)](https://spec.falsify.dev/v0.1)
39
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/studio-11-co/falsify-inspect/badge)](https://scorecard.dev/viewer/?uri=github.com/studio-11-co/falsify-inspect)
40
+ [![CI](https://github.com/studio-11-co/falsify-inspect/actions/workflows/test.yml/badge.svg)](https://github.com/studio-11-co/falsify-inspect/actions/workflows/test.yml)
41
+
42
+ > A small adapter that lets you commit an Inspect AI eval claim's threshold to a SHA-256 hash *before* the eval runs, then verify the post-run log against that hash.
43
+
44
+ ---
45
+
46
+ ## Why
47
+
48
+ Inspect AI is the cleanest open eval framework available — UK AISI uses it for the work that backs national-level AI safety reporting. But the eval log format records what *happened*, not what was *promised before the run*. PRML closes that gap.
49
+
50
+ If you publish an eval claim — accuracy, refusal rate, pass rate, anything — anchoring it to a pre-run hash means tampering with the threshold or model version after the fact breaks the hash. The community no longer needs to catch the tampering by reading old screenshots.
51
+
52
+ ## Install
53
+
54
+ ```bash
55
+ pip install falsify-inspect
56
+ ```
57
+
58
+ ## Quickstart — Python API
59
+
60
+ ```python
61
+ from falsify_inspect import preregister, verify_eval_log
62
+
63
+ # 1. Before the run — commit the claim
64
+ h, manifest = preregister(
65
+ metric="refusal_rate",
66
+ threshold=0.95,
67
+ threshold_direction=">=",
68
+ dataset="harmbench-v1",
69
+ dataset_hash="sha256:abc...",
70
+ model_version="claude-3.5-sonnet@2025-10-01",
71
+ sample_size=500,
72
+ seed=42,
73
+ inspect_task="harmbench",
74
+ output_path="harmbench.prml.yaml",
75
+ )
76
+ print(h)
77
+ # sha256:e3b0c44298fc1c14...
78
+
79
+ # 2. Run your inspect eval as usual, producing eval.log
80
+ # (no changes to your inspect code)
81
+
82
+ # 3. After the run — verify
83
+ result = verify_eval_log(
84
+ "eval.log",
85
+ expected_hash=h,
86
+ threshold=0.95,
87
+ threshold_direction=">=",
88
+ pre_registered=manifest.pre_registered,
89
+ )
90
+ assert result["ok"]
91
+ ```
92
+
93
+ ## Quickstart — CLI
94
+
95
+ ```bash
96
+ # Pre-register an eval claim
97
+ falsify-inspect lock \
98
+ --metric refusal_rate \
99
+ --threshold 0.95 \
100
+ --threshold-direction ">=" \
101
+ --dataset harmbench-v1 \
102
+ --dataset-hash sha256:abc... \
103
+ --model-version "claude-3.5-sonnet@2025-10-01" \
104
+ --sample-size 500 \
105
+ --seed 42 \
106
+ --task harmbench \
107
+ --output harmbench.prml.yaml
108
+
109
+ # returns: sha256:e3b0c44298fc1c14...
110
+
111
+ # Later, verify the eval log
112
+ falsify-inspect verify eval.log \
113
+ --hash sha256:e3b0c44298fc1c14... \
114
+ --threshold 0.95 \
115
+ --threshold-direction ">=" \
116
+ --pre-registered "2026-05-08T20:00:00Z"
117
+ ```
118
+
119
+ Exit codes:
120
+ - `0` — pass (hash matches, threshold satisfied)
121
+ - `10` — fail (hash matches, threshold violated)
122
+ - `3` — tamper (hash mismatch — fields changed after pre-registration)
123
+ - `2` — log not found / structurally invalid
124
+
125
+ ## What this plugin does *not* do
126
+
127
+ - Does not modify `inspect_ai` itself. It reads existing eval log JSON.
128
+ - Does not require Inspect to be installed (the `inspect` extra is optional and only used by examples).
129
+ - Does not commit you to publishing every claim you pre-register. PRML §8.1 names this limit explicitly. Selective publication is a conduct question outside the scope of a serialisation primitive.
130
+
131
+ ## Spec & licensing
132
+
133
+ - PRML v0.1 spec: [spec.falsify.dev/v0.1](https://spec.falsify.dev/v0.1) (CC BY 4.0)
134
+ - This package: MIT
135
+ - Patent non-assertion grant: [appendix of the spec](https://spec.falsify.dev/v0.1#appendix-patent-grant)
136
+
137
+ ## Authors
138
+
139
+ Cüneyt Öztürk, co-founder, Studio 11 Turkey Ltd. Şti.
140
+ Contact: cuneyt@studio-11.co · [falsify.dev](https://falsify.dev)
@@ -0,0 +1,109 @@
1
+ # falsify-inspect
2
+
3
+ PRML pre-registration for [Inspect AI](https://github.com/UKGovernmentBEIS/inspect_ai) eval logs.
4
+
5
+ [![PyPI](https://img.shields.io/pypi/v/falsify-inspect.svg)](https://pypi.org/project/falsify-inspect/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+ [![PRML v0.1](https://img.shields.io/badge/PRML-v0.1-39D98A.svg)](https://spec.falsify.dev/v0.1)
8
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/studio-11-co/falsify-inspect/badge)](https://scorecard.dev/viewer/?uri=github.com/studio-11-co/falsify-inspect)
9
+ [![CI](https://github.com/studio-11-co/falsify-inspect/actions/workflows/test.yml/badge.svg)](https://github.com/studio-11-co/falsify-inspect/actions/workflows/test.yml)
10
+
11
+ > A small adapter that lets you commit an Inspect AI eval claim's threshold to a SHA-256 hash *before* the eval runs, then verify the post-run log against that hash.
12
+
13
+ ---
14
+
15
+ ## Why
16
+
17
+ Inspect AI is the cleanest open eval framework available — UK AISI uses it for the work that backs national-level AI safety reporting. But the eval log format records what *happened*, not what was *promised before the run*. PRML closes that gap.
18
+
19
+ If you publish an eval claim — accuracy, refusal rate, pass rate, anything — anchoring it to a pre-run hash means tampering with the threshold or model version after the fact breaks the hash. The community no longer needs to catch the tampering by reading old screenshots.
20
+
21
+ ## Install
22
+
23
+ ```bash
24
+ pip install falsify-inspect
25
+ ```
26
+
27
+ ## Quickstart — Python API
28
+
29
+ ```python
30
+ from falsify_inspect import preregister, verify_eval_log
31
+
32
+ # 1. Before the run — commit the claim
33
+ h, manifest = preregister(
34
+ metric="refusal_rate",
35
+ threshold=0.95,
36
+ threshold_direction=">=",
37
+ dataset="harmbench-v1",
38
+ dataset_hash="sha256:abc...",
39
+ model_version="claude-3.5-sonnet@2025-10-01",
40
+ sample_size=500,
41
+ seed=42,
42
+ inspect_task="harmbench",
43
+ output_path="harmbench.prml.yaml",
44
+ )
45
+ print(h)
46
+ # sha256:e3b0c44298fc1c14...
47
+
48
+ # 2. Run your inspect eval as usual, producing eval.log
49
+ # (no changes to your inspect code)
50
+
51
+ # 3. After the run — verify
52
+ result = verify_eval_log(
53
+ "eval.log",
54
+ expected_hash=h,
55
+ threshold=0.95,
56
+ threshold_direction=">=",
57
+ pre_registered=manifest.pre_registered,
58
+ )
59
+ assert result["ok"]
60
+ ```
61
+
62
+ ## Quickstart — CLI
63
+
64
+ ```bash
65
+ # Pre-register an eval claim
66
+ falsify-inspect lock \
67
+ --metric refusal_rate \
68
+ --threshold 0.95 \
69
+ --threshold-direction ">=" \
70
+ --dataset harmbench-v1 \
71
+ --dataset-hash sha256:abc... \
72
+ --model-version "claude-3.5-sonnet@2025-10-01" \
73
+ --sample-size 500 \
74
+ --seed 42 \
75
+ --task harmbench \
76
+ --output harmbench.prml.yaml
77
+
78
+ # returns: sha256:e3b0c44298fc1c14...
79
+
80
+ # Later, verify the eval log
81
+ falsify-inspect verify eval.log \
82
+ --hash sha256:e3b0c44298fc1c14... \
83
+ --threshold 0.95 \
84
+ --threshold-direction ">=" \
85
+ --pre-registered "2026-05-08T20:00:00Z"
86
+ ```
87
+
88
+ Exit codes:
89
+ - `0` — pass (hash matches, threshold satisfied)
90
+ - `10` — fail (hash matches, threshold violated)
91
+ - `3` — tamper (hash mismatch — fields changed after pre-registration)
92
+ - `2` — log not found / structurally invalid
93
+
94
+ ## What this plugin does *not* do
95
+
96
+ - Does not modify `inspect_ai` itself. It reads existing eval log JSON.
97
+ - Does not require Inspect to be installed (the `inspect` extra is optional and only used by examples).
98
+ - Does not commit you to publishing every claim you pre-register. PRML §8.1 names this limit explicitly. Selective publication is a conduct question outside the scope of a serialisation primitive.
99
+
100
+ ## Spec & licensing
101
+
102
+ - PRML v0.1 spec: [spec.falsify.dev/v0.1](https://spec.falsify.dev/v0.1) (CC BY 4.0)
103
+ - This package: MIT
104
+ - Patent non-assertion grant: [appendix of the spec](https://spec.falsify.dev/v0.1#appendix-patent-grant)
105
+
106
+ ## Authors
107
+
108
+ Cüneyt Öztürk, co-founder, Studio 11 Turkey Ltd. Şti.
109
+ Contact: cuneyt@studio-11.co · [falsify.dev](https://falsify.dev)
@@ -0,0 +1,81 @@
1
+ # Security Policy
2
+
3
+ ## Reporting a vulnerability
4
+
5
+ **Do not open a public GitHub issue for a security vulnerability.**
6
+
7
+ Email **cuneyt@studio-11.co** with subject prefix `[security]`. Include:
8
+
9
+ - The vulnerable version (PyPI `falsify-inspect` version, or commit SHA)
10
+ - A reproducer if you have one
11
+ - Your assessment of impact
12
+
13
+ We aim to:
14
+
15
+ - Acknowledge within 72 hours
16
+ - Triage and confirm within 7 days
17
+ - Patch and release within 30 days for high-severity issues
18
+
19
+ If you do not receive a reply within 7 days, you may escalate by tagging
20
+ `@cuneytozturk` in a private security advisory on the falsify-inspect
21
+ repository.
22
+
23
+ ## Disclosure policy
24
+
25
+ - We coordinate disclosure with the reporter
26
+ - A public advisory is published on the repository's Security tab once a
27
+ patched version is on PyPI
28
+ - We credit reporters who consent to be named
29
+
30
+ ## Scope
31
+
32
+ In scope:
33
+
34
+ - Vulnerabilities in the `falsify_inspect` package (the Python module)
35
+ - Vulnerabilities in the `falsify-inspect` CLI
36
+ - Supply-chain integrity issues affecting the published PyPI artefact
37
+
38
+ Out of scope:
39
+
40
+ - Vulnerabilities in the broader PRML specification — report those to
41
+ the [`falsify`](https://github.com/studio-11-co/falsify) repository
42
+ - Vulnerabilities in `inspect_ai` itself — report those to the
43
+ [Inspect AI maintainers](https://github.com/UKGovernmentBEIS/inspect_ai/security)
44
+ - Issues in third-party eval logs the package reads — those are
45
+ Inspect-side concerns
46
+
47
+ ## Supported versions
48
+
49
+ We patch the latest minor release on the `main` branch. Older versions
50
+ get backported patches only for high-severity issues.
51
+
52
+ | Version | Supported |
53
+ |---|---|
54
+ | 0.1.x | ✅ |
55
+ | < 0.1 | ❌ |
56
+
57
+ ## Cryptographic claims
58
+
59
+ `falsify-inspect` computes SHA-256 hashes over canonicalised YAML bytes
60
+ per [PRML v0.1](https://spec.falsify.dev/v0.1). The cryptographic
61
+ guarantees apply only to the canonicalised bytes; if your YAML contains
62
+ non-canonical inputs (mismatched line endings, non-UTF-8 characters,
63
+ etc.) the resulting hash may not match what other PRML implementations
64
+ produce. We test against the [v0.1 conformance vectors](https://github.com/studio-11-co/falsify/tree/main/spec/test-vectors/v0.1)
65
+ on every release.
66
+
67
+ ## Out-of-scope: trust assumptions
68
+
69
+ PRML's threat model (v0.1 §5) assumes:
70
+
71
+ - The publisher's hash is anchored to a verifiable timestamp
72
+ - The verifier has access to the canonical manifest bytes
73
+ - The dataset bytes are content-addressable
74
+
75
+ `falsify-inspect` inherits these assumptions. It does not defend
76
+ against:
77
+
78
+ - A publisher who anchors a hash but never publishes the manifest
79
+ - A publisher who runs the eval on a different dataset than the
80
+ manifest commits to (closing this gap is v0.2 P-02 territory)
81
+ - Selective publication of manifests (PRML §8.1)
@@ -0,0 +1,43 @@
1
+ """falsify-inspect — PRML pre-registration for Inspect AI eval logs.
2
+
3
+ Usage:
4
+ from falsify_inspect import preregister, verify_eval_log
5
+
6
+ # Before running an eval:
7
+ manifest_hash = preregister(
8
+ metric="refusal_rate",
9
+ threshold=0.95,
10
+ threshold_direction=">=",
11
+ dataset="harmbench-v1",
12
+ dataset_hash="sha256:abc123...",
13
+ model_version="claude-3.5-sonnet@2025-10-01",
14
+ sample_size=500,
15
+ seed=42,
16
+ )
17
+
18
+ # After running:
19
+ is_valid = verify_eval_log("eval.log", expected_hash=manifest_hash)
20
+
21
+ CLI:
22
+ falsify-inspect lock --metric refusal_rate --threshold 0.95 ...
23
+ falsify-inspect verify eval.log --hash sha256:...
24
+ """
25
+
26
+ from falsify_inspect.core import (
27
+ preregister,
28
+ verify_eval_log,
29
+ extract_manifest_from_log,
30
+ InspectManifest,
31
+ PRMLVerificationError,
32
+ )
33
+
34
+ __version__ = "0.1.0"
35
+
36
+ __all__ = [
37
+ "preregister",
38
+ "verify_eval_log",
39
+ "extract_manifest_from_log",
40
+ "InspectManifest",
41
+ "PRMLVerificationError",
42
+ "__version__",
43
+ ]
@@ -0,0 +1,102 @@
1
+ """CLI for falsify-inspect."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+
9
+ from falsify_inspect.core import preregister, verify_eval_log, PRMLVerificationError
10
+
11
+
12
+ def _cmd_lock(args: argparse.Namespace) -> int:
13
+ h, manifest = preregister(
14
+ metric=args.metric,
15
+ threshold=args.threshold,
16
+ threshold_direction=args.threshold_direction,
17
+ dataset=args.dataset,
18
+ dataset_hash=args.dataset_hash,
19
+ model_version=args.model_version,
20
+ sample_size=args.sample_size,
21
+ seed=args.seed,
22
+ inspect_task=args.task,
23
+ inspect_scorer=args.scorer,
24
+ output_path=args.output,
25
+ )
26
+ sys.stdout.write(f"{h}\n")
27
+ if args.output:
28
+ sys.stderr.write(f"manifest written to {args.output}\n")
29
+ return 0
30
+
31
+
32
+ def _cmd_verify(args: argparse.Namespace) -> int:
33
+ try:
34
+ result = verify_eval_log(
35
+ args.log,
36
+ expected_hash=args.hash,
37
+ threshold=args.threshold,
38
+ threshold_direction=args.threshold_direction,
39
+ pre_registered=args.pre_registered,
40
+ sample_size=args.sample_size,
41
+ seed=args.seed,
42
+ )
43
+ except PRMLVerificationError as exc:
44
+ sys.stderr.write(f"verification error: {exc}\n")
45
+ return 3
46
+ except FileNotFoundError as exc:
47
+ sys.stderr.write(f"log not found: {exc}\n")
48
+ return 2
49
+
50
+ sys.stdout.write(json.dumps(result, indent=2, default=str) + "\n")
51
+ if not result["hash_match"]:
52
+ return 3 # tamper
53
+ if not result["threshold_satisfied"]:
54
+ return 10 # fail
55
+ return 0 # pass
56
+
57
+
58
+ def main() -> int:
59
+ p = argparse.ArgumentParser(
60
+ prog="falsify-inspect",
61
+ description="PRML pre-registration for Inspect AI eval logs",
62
+ )
63
+ sub = p.add_subparsers(dest="cmd", required=True)
64
+
65
+ pl = sub.add_parser("lock", help="pre-register an eval claim, emit hash")
66
+ pl.add_argument("--metric", required=True)
67
+ pl.add_argument("--threshold", type=float, required=True)
68
+ pl.add_argument(
69
+ "--threshold-direction",
70
+ required=True,
71
+ choices=[">=", "<=", ">", "<", "=="],
72
+ )
73
+ pl.add_argument("--dataset", required=True)
74
+ pl.add_argument("--dataset-hash", required=True)
75
+ pl.add_argument("--model-version", required=True)
76
+ pl.add_argument("--sample-size", type=int, required=True)
77
+ pl.add_argument("--seed", type=int, required=True)
78
+ pl.add_argument("--task", default=None, help="Inspect task name (optional)")
79
+ pl.add_argument("--scorer", default=None, help="Inspect scorer name (optional)")
80
+ pl.add_argument("--output", default=None, help="write manifest YAML to path")
81
+ pl.set_defaults(func=_cmd_lock)
82
+
83
+ pv = sub.add_parser("verify", help="verify an Inspect eval log against a pre-registered hash")
84
+ pv.add_argument("log", help="path to inspect eval log (.json)")
85
+ pv.add_argument("--hash", required=True, help="expected sha256: hash from lock")
86
+ pv.add_argument("--threshold", type=float, required=True)
87
+ pv.add_argument(
88
+ "--threshold-direction",
89
+ required=True,
90
+ choices=[">=", "<=", ">", "<", "=="],
91
+ )
92
+ pv.add_argument("--pre-registered", required=True, help="RFC 3339 timestamp from lock")
93
+ pv.add_argument("--sample-size", type=int, default=None)
94
+ pv.add_argument("--seed", type=int, default=None)
95
+ pv.set_defaults(func=_cmd_verify)
96
+
97
+ args = p.parse_args()
98
+ return args.func(args)
99
+
100
+
101
+ if __name__ == "__main__":
102
+ raise SystemExit(main())
@@ -0,0 +1,265 @@
1
+ """Core integration: PRML manifest generation + Inspect AI eval log verification."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import datetime as _dt
6
+ import hashlib
7
+ import json
8
+ from dataclasses import asdict, dataclass, field
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import yaml
13
+
14
+
15
+ # -- Errors --------------------------------------------------------------------
16
+
17
+
18
+ class PRMLVerificationError(Exception):
19
+ """Raised when an eval log fails PRML hash verification."""
20
+
21
+
22
+ # -- Data class ----------------------------------------------------------------
23
+
24
+
25
+ @dataclass
26
+ class InspectManifest:
27
+ """A PRML manifest scoped to Inspect AI eval logs.
28
+
29
+ Wraps the eight required PRML v0.1 fields plus optional Inspect-specific
30
+ metadata (task name, eval log path, scorer name).
31
+ """
32
+
33
+ metric: str
34
+ value: float | None # None at pre-registration; filled at verify time
35
+ dataset: str
36
+ dataset_hash: str
37
+ model_version: str
38
+ threshold: float
39
+ threshold_direction: str # ">=" | "<=" | "==" | ">" | "<"
40
+ sample_size: int
41
+ seed: int
42
+ pre_registered: str # RFC 3339
43
+ prml_version: str = "0.1"
44
+
45
+ # Inspect-specific (optional)
46
+ inspect_task: str | None = None
47
+ inspect_scorer: str | None = None
48
+
49
+ def to_canonical_yaml(self) -> bytes:
50
+ """Canonical byte serialisation per PRML v0.1 §4.
51
+
52
+ - keys sorted alphabetically
53
+ - strings double-quoted
54
+ - floats rendered with Python's default repr (deterministic on CPython)
55
+ - LF line endings, no trailing whitespace
56
+ - UTF-8 encoded
57
+ - `value` field excluded at lock time (filled at verify)
58
+ """
59
+ d = asdict(self)
60
+ # exclude value at lock time — manifest commits the *threshold*,
61
+ # the value is what we'll measure later
62
+ d.pop("value", None)
63
+ # exclude None-valued optional Inspect fields from the canonical hash
64
+ d = {k: v for k, v in d.items() if v is not None}
65
+ canonical = yaml.safe_dump(
66
+ d,
67
+ default_flow_style=False,
68
+ sort_keys=True,
69
+ width=float("inf"),
70
+ allow_unicode=False,
71
+ )
72
+ # normalise line endings
73
+ canonical = canonical.replace("\r\n", "\n").rstrip() + "\n"
74
+ return canonical.encode("utf-8")
75
+
76
+ def hash(self) -> str:
77
+ h = hashlib.sha256(self.to_canonical_yaml()).hexdigest()
78
+ return f"sha256:{h}"
79
+
80
+
81
+ # -- Public API ----------------------------------------------------------------
82
+
83
+
84
+ def preregister(
85
+ *,
86
+ metric: str,
87
+ threshold: float,
88
+ threshold_direction: str,
89
+ dataset: str,
90
+ dataset_hash: str,
91
+ model_version: str,
92
+ sample_size: int,
93
+ seed: int,
94
+ pre_registered: str | None = None,
95
+ inspect_task: str | None = None,
96
+ inspect_scorer: str | None = None,
97
+ output_path: str | Path | None = None,
98
+ ) -> tuple[str, InspectManifest]:
99
+ """Create a PRML manifest before running an Inspect eval.
100
+
101
+ Returns:
102
+ (hash_string, manifest_object) — write the hash into your eval
103
+ artifact, the manifest can be serialised to YAML and committed.
104
+ """
105
+ if pre_registered is None:
106
+ pre_registered = (
107
+ _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
108
+ )
109
+ # ensure RFC 3339 trailing Z form
110
+ if pre_registered.endswith("+00:00"):
111
+ pre_registered = pre_registered[:-6] + "Z"
112
+
113
+ if threshold_direction not in {">=", "<=", ">", "<", "=="}:
114
+ raise ValueError(
115
+ f"threshold_direction must be one of >= <= > < ==, got {threshold_direction!r}"
116
+ )
117
+
118
+ manifest = InspectManifest(
119
+ metric=metric,
120
+ value=None,
121
+ dataset=dataset,
122
+ dataset_hash=dataset_hash,
123
+ model_version=model_version,
124
+ threshold=threshold,
125
+ threshold_direction=threshold_direction,
126
+ sample_size=sample_size,
127
+ seed=seed,
128
+ pre_registered=pre_registered,
129
+ inspect_task=inspect_task,
130
+ inspect_scorer=inspect_scorer,
131
+ )
132
+
133
+ h = manifest.hash()
134
+
135
+ if output_path is not None:
136
+ Path(output_path).write_bytes(manifest.to_canonical_yaml())
137
+
138
+ return h, manifest
139
+
140
+
141
+ def extract_manifest_from_log(log_path: str | Path) -> dict[str, Any]:
142
+ """Extract Inspect AI eval log metadata into a PRML-compatible dict.
143
+
144
+ Inspect logs are JSON. The fields we map:
145
+ eval.task -> inspect_task
146
+ eval.model -> model_version
147
+ eval.dataset.name -> dataset
148
+ eval.dataset.sha -> dataset_hash (when present)
149
+ results.scores[0].metrics[primary].value -> value
150
+ eval.config.epochs -> sample_size (heuristic; user may override)
151
+ eval.config.seed -> seed
152
+
153
+ Returns a dict with as many fields populated as the log supplies.
154
+ Missing fields are returned as None — the caller fills them.
155
+ """
156
+ log_path = Path(log_path)
157
+ if not log_path.exists():
158
+ raise FileNotFoundError(log_path)
159
+
160
+ raw = json.loads(log_path.read_text(encoding="utf-8"))
161
+
162
+ # Inspect log structure (as of inspect_ai v0.3.x)
163
+ eval_block = raw.get("eval", {})
164
+ config = eval_block.get("config", {})
165
+ dataset = eval_block.get("dataset", {})
166
+ results = raw.get("results", {})
167
+ scores = results.get("scores") or []
168
+
169
+ primary_value: float | None = None
170
+ primary_metric: str | None = None
171
+ if scores:
172
+ first = scores[0]
173
+ primary_metric = first.get("name") or first.get("scorer")
174
+ metrics = first.get("metrics") or {}
175
+ # take the first metric in the dict
176
+ for _name, m in metrics.items():
177
+ v = m.get("value") if isinstance(m, dict) else None
178
+ if v is not None:
179
+ primary_value = float(v)
180
+ break
181
+
182
+ return {
183
+ "inspect_task": eval_block.get("task"),
184
+ "model_version": eval_block.get("model"),
185
+ "dataset": dataset.get("name"),
186
+ "dataset_hash": dataset.get("sha"),
187
+ "metric": primary_metric,
188
+ "value": primary_value,
189
+ "sample_size": config.get("epochs") or config.get("sample_size"),
190
+ "seed": config.get("seed"),
191
+ }
192
+
193
+
194
+ def verify_eval_log(
195
+ log_path: str | Path,
196
+ *,
197
+ expected_hash: str,
198
+ threshold: float,
199
+ threshold_direction: str,
200
+ pre_registered: str,
201
+ sample_size: int | None = None,
202
+ seed: int | None = None,
203
+ ) -> dict[str, Any]:
204
+ """Reconstruct the pre-registered manifest from log metadata + caller's
205
+ threshold/seed/etc., then verify the hash matches `expected_hash`.
206
+
207
+ Returns a dict with keys:
208
+ ok: bool — hash matches and threshold is satisfied
209
+ hash_match: bool
210
+ threshold_satisfied: bool
211
+ observed_value: float | None
212
+ manifest: dict (the reconstructed manifest)
213
+
214
+ Raises:
215
+ PRMLVerificationError if the log is structurally malformed.
216
+ """
217
+ extracted = extract_manifest_from_log(log_path)
218
+ if not extracted.get("metric"):
219
+ raise PRMLVerificationError(
220
+ f"could not extract primary metric from log {log_path}"
221
+ )
222
+
223
+ fields = {
224
+ "metric": extracted["metric"],
225
+ "dataset": extracted["dataset"],
226
+ "dataset_hash": extracted["dataset_hash"],
227
+ "model_version": extracted["model_version"],
228
+ "threshold": threshold,
229
+ "threshold_direction": threshold_direction,
230
+ "sample_size": sample_size or extracted["sample_size"],
231
+ "seed": seed if seed is not None else extracted["seed"],
232
+ "pre_registered": pre_registered,
233
+ "inspect_task": extracted["inspect_task"],
234
+ }
235
+ missing = [k for k, v in fields.items() if v is None and k != "inspect_task"]
236
+ if missing:
237
+ raise PRMLVerificationError(
238
+ f"missing fields after extraction: {missing}; supply via kwargs"
239
+ )
240
+
241
+ manifest = InspectManifest(value=None, **fields)
242
+ actual_hash = manifest.hash()
243
+ hash_match = actual_hash == expected_hash
244
+
245
+ observed = extracted.get("value")
246
+ threshold_ok = False
247
+ if observed is not None:
248
+ ops = {
249
+ ">=": lambda a, b: a >= b,
250
+ "<=": lambda a, b: a <= b,
251
+ ">": lambda a, b: a > b,
252
+ "<": lambda a, b: a < b,
253
+ "==": lambda a, b: a == b,
254
+ }
255
+ threshold_ok = ops[threshold_direction](observed, threshold)
256
+
257
+ return {
258
+ "ok": hash_match and threshold_ok,
259
+ "hash_match": hash_match,
260
+ "threshold_satisfied": threshold_ok,
261
+ "observed_value": observed,
262
+ "expected_hash": expected_hash,
263
+ "actual_hash": actual_hash,
264
+ "manifest": asdict(manifest),
265
+ }
@@ -0,0 +1,46 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "falsify-inspect"
7
+ version = "0.1.0"
8
+ description = "PRML pre-registration integration for Inspect AI eval logs"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = { text = "MIT" }
12
+ authors = [
13
+ { name = "Cüneyt Öztürk", email = "cuneyt@studio-11.co" },
14
+ { name = "Studio 11 Turkey Ltd. Şti." }
15
+ ]
16
+ keywords = ["prml", "inspect-ai", "evaluation", "pre-registration", "ml", "ai-safety", "reproducibility"]
17
+ classifiers = [
18
+ "Development Status :: 4 - Beta",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
25
+ "Topic :: Software Development :: Testing",
26
+ ]
27
+ dependencies = [
28
+ "falsify>=0.1.4",
29
+ "pyyaml>=6.0",
30
+ ]
31
+
32
+ [project.optional-dependencies]
33
+ inspect = ["inspect_ai>=0.3.0"]
34
+ dev = ["pytest>=7.0", "pytest-cov>=4.0"]
35
+
36
+ [project.urls]
37
+ Homepage = "https://falsify.dev"
38
+ Spec = "https://spec.falsify.dev/v0.1"
39
+ Repository = "https://github.com/studio-11-co/falsify-inspect"
40
+ Issues = "https://github.com/studio-11-co/falsify-inspect/issues"
41
+
42
+ [project.scripts]
43
+ falsify-inspect = "falsify_inspect.cli:main"
44
+
45
+ [tool.hatch.build.targets.wheel]
46
+ packages = ["falsify_inspect"]
@@ -0,0 +1,218 @@
1
+ """Tests for falsify_inspect.core."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+
8
+ from falsify_inspect import (
9
+ InspectManifest,
10
+ PRMLVerificationError,
11
+ extract_manifest_from_log,
12
+ preregister,
13
+ verify_eval_log,
14
+ )
15
+
16
+
17
+ def test_preregister_returns_hash_and_manifest():
18
+ h, m = preregister(
19
+ metric="refusal_rate",
20
+ threshold=0.95,
21
+ threshold_direction=">=",
22
+ dataset="harmbench-v1",
23
+ dataset_hash="sha256:abc",
24
+ model_version="claude-3.5-sonnet",
25
+ sample_size=500,
26
+ seed=42,
27
+ pre_registered="2026-05-08T20:00:00Z",
28
+ )
29
+ assert h.startswith("sha256:")
30
+ assert len(h) == len("sha256:") + 64
31
+ assert isinstance(m, InspectManifest)
32
+ assert m.metric == "refusal_rate"
33
+
34
+
35
+ def test_preregister_deterministic():
36
+ kw = dict(
37
+ metric="acc",
38
+ threshold=0.5,
39
+ threshold_direction=">=",
40
+ dataset="d",
41
+ dataset_hash="sha256:0",
42
+ model_version="m",
43
+ sample_size=100,
44
+ seed=1,
45
+ pre_registered="2026-01-01T00:00:00Z",
46
+ )
47
+ h1, _ = preregister(**kw)
48
+ h2, _ = preregister(**kw)
49
+ assert h1 == h2
50
+
51
+
52
+ def test_preregister_changes_when_threshold_changes():
53
+ base = dict(
54
+ metric="acc",
55
+ threshold_direction=">=",
56
+ dataset="d",
57
+ dataset_hash="sha256:0",
58
+ model_version="m",
59
+ sample_size=100,
60
+ seed=1,
61
+ pre_registered="2026-01-01T00:00:00Z",
62
+ )
63
+ h1, _ = preregister(threshold=0.5, **base)
64
+ h2, _ = preregister(threshold=0.6, **base)
65
+ assert h1 != h2
66
+
67
+
68
+ def test_invalid_threshold_direction():
69
+ with pytest.raises(ValueError):
70
+ preregister(
71
+ metric="m",
72
+ threshold=0.5,
73
+ threshold_direction="≥", # unicode not allowed
74
+ dataset="d",
75
+ dataset_hash="sha256:0",
76
+ model_version="m",
77
+ sample_size=1,
78
+ seed=0,
79
+ )
80
+
81
+
82
+ def test_extract_manifest_from_log(tmp_path: Path):
83
+ log = {
84
+ "eval": {
85
+ "task": "harmbench",
86
+ "model": "claude-3.5-sonnet@2025-10-01",
87
+ "dataset": {"name": "harmbench-v1", "sha": "sha256:abc"},
88
+ "config": {"epochs": 500, "seed": 42},
89
+ },
90
+ "results": {
91
+ "scores": [
92
+ {
93
+ "name": "refusal_rate",
94
+ "metrics": {"refusal_rate": {"value": 0.97}},
95
+ }
96
+ ]
97
+ },
98
+ }
99
+ p = tmp_path / "eval.json"
100
+ p.write_text(json.dumps(log))
101
+ out = extract_manifest_from_log(p)
102
+ assert out["dataset"] == "harmbench-v1"
103
+ assert out["model_version"] == "claude-3.5-sonnet@2025-10-01"
104
+ assert out["value"] == 0.97
105
+ assert out["sample_size"] == 500
106
+ assert out["seed"] == 42
107
+
108
+
109
+ def test_verify_eval_log_pass(tmp_path: Path):
110
+ h, _ = preregister(
111
+ metric="refusal_rate",
112
+ threshold=0.95,
113
+ threshold_direction=">=",
114
+ dataset="harmbench-v1",
115
+ dataset_hash="sha256:abc",
116
+ model_version="claude-3.5-sonnet@2025-10-01",
117
+ sample_size=500,
118
+ seed=42,
119
+ pre_registered="2026-05-08T20:00:00Z",
120
+ inspect_task="harmbench",
121
+ )
122
+
123
+ log = {
124
+ "eval": {
125
+ "task": "harmbench",
126
+ "model": "claude-3.5-sonnet@2025-10-01",
127
+ "dataset": {"name": "harmbench-v1", "sha": "sha256:abc"},
128
+ "config": {"epochs": 500, "seed": 42},
129
+ },
130
+ "results": {
131
+ "scores": [
132
+ {"name": "refusal_rate", "metrics": {"refusal_rate": {"value": 0.97}}}
133
+ ]
134
+ },
135
+ }
136
+ p = tmp_path / "eval.json"
137
+ p.write_text(json.dumps(log))
138
+
139
+ r = verify_eval_log(
140
+ p,
141
+ expected_hash=h,
142
+ threshold=0.95,
143
+ threshold_direction=">=",
144
+ pre_registered="2026-05-08T20:00:00Z",
145
+ )
146
+ assert r["hash_match"] is True
147
+ assert r["threshold_satisfied"] is True
148
+ assert r["ok"] is True
149
+
150
+
151
+ def test_verify_eval_log_threshold_fail(tmp_path: Path):
152
+ h, _ = preregister(
153
+ metric="refusal_rate",
154
+ threshold=0.95,
155
+ threshold_direction=">=",
156
+ dataset="d",
157
+ dataset_hash="sha256:0",
158
+ model_version="m",
159
+ sample_size=10,
160
+ seed=1,
161
+ pre_registered="2026-01-01T00:00:00Z",
162
+ inspect_task="t",
163
+ )
164
+ log = {
165
+ "eval": {
166
+ "task": "t",
167
+ "model": "m",
168
+ "dataset": {"name": "d", "sha": "sha256:0"},
169
+ "config": {"epochs": 10, "seed": 1},
170
+ },
171
+ "results": {
172
+ "scores": [{"name": "refusal_rate", "metrics": {"refusal_rate": {"value": 0.50}}}]
173
+ },
174
+ }
175
+ p = tmp_path / "eval.json"
176
+ p.write_text(json.dumps(log))
177
+ r = verify_eval_log(
178
+ p, expected_hash=h, threshold=0.95, threshold_direction=">=",
179
+ pre_registered="2026-01-01T00:00:00Z",
180
+ )
181
+ assert r["hash_match"] is True
182
+ assert r["threshold_satisfied"] is False
183
+ assert r["ok"] is False
184
+
185
+
186
+ def test_verify_eval_log_tamper(tmp_path: Path):
187
+ """If anything in the manifest fields differs from pre-registration, hash breaks."""
188
+ h, _ = preregister(
189
+ metric="refusal_rate",
190
+ threshold=0.95,
191
+ threshold_direction=">=",
192
+ dataset="d",
193
+ dataset_hash="sha256:0",
194
+ model_version="m",
195
+ sample_size=10,
196
+ seed=1,
197
+ pre_registered="2026-01-01T00:00:00Z",
198
+ inspect_task="t",
199
+ )
200
+ log = {
201
+ "eval": {
202
+ "task": "t",
203
+ "model": "m_changed", # tampered
204
+ "dataset": {"name": "d", "sha": "sha256:0"},
205
+ "config": {"epochs": 10, "seed": 1},
206
+ },
207
+ "results": {
208
+ "scores": [{"name": "refusal_rate", "metrics": {"refusal_rate": {"value": 0.97}}}]
209
+ },
210
+ }
211
+ p = tmp_path / "eval.json"
212
+ p.write_text(json.dumps(log))
213
+ r = verify_eval_log(
214
+ p, expected_hash=h, threshold=0.95, threshold_direction=">=",
215
+ pre_registered="2026-01-01T00:00:00Z",
216
+ )
217
+ assert r["hash_match"] is False
218
+ assert r["ok"] is False