results-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- results_cli-0.1.0/.github/workflows/publish.yml +21 -0
- results_cli-0.1.0/.gitignore +6 -0
- results_cli-0.1.0/LICENSE +21 -0
- results_cli-0.1.0/PKG-INFO +114 -0
- results_cli-0.1.0/README.md +103 -0
- results_cli-0.1.0/plugin/.claude-plugin/plugin.json +9 -0
- results_cli-0.1.0/plugin/README.md +31 -0
- results_cli-0.1.0/plugin/skills/results/SKILL.md +99 -0
- results_cli-0.1.0/pyproject.toml +20 -0
- results_cli-0.1.0/src/results/__init__.py +3 -0
- results_cli-0.1.0/src/results/cli.py +277 -0
- results_cli-0.1.0/src/results/ledger.py +78 -0
- results_cli-0.1.0/tests/test_cli.py +133 -0
- results_cli-0.1.0/tests/test_ledger.py +89 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
publish:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
environment: pypi
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write
|
|
13
|
+
contents: read
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.11"
|
|
19
|
+
- run: pip install hatchling
|
|
20
|
+
- run: python -m hatchling build
|
|
21
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Elliot Tower
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: results-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Seal a run, record what it produced, and verify the chain
|
|
5
|
+
Author-email: Elliot Tower <elliot@elliottower.ai>
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: open science,provenance,reproducibility,results
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# results
|
|
13
|
+
|
|
14
|
+
Seal a run, record what it produced, and verify the chain.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install results-cli
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Quick start
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
results init
|
|
26
|
+
results seal prereg.md analysis.py data.csv --role input
|
|
27
|
+
results access "read zenodo metadata" --level "metadata only"
|
|
28
|
+
|
|
29
|
+
# run the computation, then record its outputs
|
|
30
|
+
results run output.json --run-id exp_001 --note "ICC analysis"
|
|
31
|
+
results claim "ICC = 0.42" --run-id exp_001 --confirmatory --location "Table 2"
|
|
32
|
+
results verify --files
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
chain intact: 5 events
|
|
37
|
+
|
|
38
|
+
access 1
|
|
39
|
+
claim 1
|
|
40
|
+
init 1
|
|
41
|
+
run 1
|
|
42
|
+
seal 1
|
|
43
|
+
|
|
44
|
+
file hashes:
|
|
45
|
+
ok prereg.md
|
|
46
|
+
ok analysis.py
|
|
47
|
+
ok data.csv
|
|
48
|
+
ok output.json
|
|
49
|
+
|
|
50
|
+
all checks passed.
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Commands
|
|
54
|
+
|
|
55
|
+
| Command | What it does |
|
|
56
|
+
|---------|-------------|
|
|
57
|
+
| `results init` | Start tracking results here |
|
|
58
|
+
| `results seal <file>...` | Hash inputs before a run |
|
|
59
|
+
| `results access <note>` | Record a data-access event |
|
|
60
|
+
| `results run <file>...` | Record outputs after a run |
|
|
61
|
+
| `results claim <text>` | Bind a manuscript claim to a run |
|
|
62
|
+
| `results verify` | Check the ledger chain and every hash it names |
|
|
63
|
+
|
|
64
|
+
## The chain
|
|
65
|
+
|
|
66
|
+
A number in a manuscript names a claim. The claim names a run. The run names its outputs. The
|
|
67
|
+
outputs were hashed when they were recorded. The inputs were hashed before the run started.
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
manuscript → claim → run → output file → sha256
|
|
71
|
+
input files → sha256
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
`results verify --files` walks the whole thing and tells you what moved.
|
|
75
|
+
|
|
76
|
+
## Data-access levels
|
|
77
|
+
|
|
78
|
+
The access timeline is what makes the confirmatory/exploratory distinction verifiable.
|
|
79
|
+
|
|
80
|
+
| Level | Meaning |
|
|
81
|
+
|-------|---------|
|
|
82
|
+
| `nothing seen` | No target data touched |
|
|
83
|
+
| `metadata only` | Structure, region names, sample sizes — not outcomes |
|
|
84
|
+
| `structure seen` | Data shape and distributions, not the target variable |
|
|
85
|
+
| `outcomes seen` | The dependent variable was observed |
|
|
86
|
+
|
|
87
|
+
An analysis registered after `outcomes seen` is retrospective.
|
|
88
|
+
|
|
89
|
+
## Verify output
|
|
90
|
+
|
|
91
|
+
| Result | Meaning |
|
|
92
|
+
|--------|---------|
|
|
93
|
+
| `chain intact` | Every event's prev_hash matches the line before it |
|
|
94
|
+
| `CHAIN BROKEN` | The ledger was edited after it was written |
|
|
95
|
+
| `ok` | File matches its recorded hash |
|
|
96
|
+
| `CHANGED` | File was modified since it was recorded |
|
|
97
|
+
| `MISSING` | File no longer exists |
|
|
98
|
+
|
|
99
|
+
## The ledger
|
|
100
|
+
|
|
101
|
+
Append-only JSONL in `.results/ledger.jsonl`. Each line is hash-chained to the previous — editing
|
|
102
|
+
or inserting a line breaks the chain. `git diff` shows what changed; `results verify` checks
|
|
103
|
+
whether it should have.
|
|
104
|
+
|
|
105
|
+
## Claude Code
|
|
106
|
+
|
|
107
|
+
`plugin/` is a Claude Code plugin that tells Claude when to reach for the CLI.
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
/plugin marketplace add elliottower/results
|
|
111
|
+
/plugin install results@results
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
MIT licensed.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# results
|
|
2
|
+
|
|
3
|
+
Seal a run, record what it produced, and verify the chain.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install results-cli
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
results init
|
|
15
|
+
results seal prereg.md analysis.py data.csv --role input
|
|
16
|
+
results access "read zenodo metadata" --level "metadata only"
|
|
17
|
+
|
|
18
|
+
# run the computation, then record its outputs
|
|
19
|
+
results run output.json --run-id exp_001 --note "ICC analysis"
|
|
20
|
+
results claim "ICC = 0.42" --run-id exp_001 --confirmatory --location "Table 2"
|
|
21
|
+
results verify --files
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
chain intact: 5 events
|
|
26
|
+
|
|
27
|
+
access 1
|
|
28
|
+
claim 1
|
|
29
|
+
init 1
|
|
30
|
+
run 1
|
|
31
|
+
seal 1
|
|
32
|
+
|
|
33
|
+
file hashes:
|
|
34
|
+
ok prereg.md
|
|
35
|
+
ok analysis.py
|
|
36
|
+
ok data.csv
|
|
37
|
+
ok output.json
|
|
38
|
+
|
|
39
|
+
all checks passed.
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Commands
|
|
43
|
+
|
|
44
|
+
| Command | What it does |
|
|
45
|
+
|---------|-------------|
|
|
46
|
+
| `results init` | Start tracking results here |
|
|
47
|
+
| `results seal <file>...` | Hash inputs before a run |
|
|
48
|
+
| `results access <note>` | Record a data-access event |
|
|
49
|
+
| `results run <file>...` | Record outputs after a run |
|
|
50
|
+
| `results claim <text>` | Bind a manuscript claim to a run |
|
|
51
|
+
| `results verify` | Check the ledger chain and every hash it names |
|
|
52
|
+
|
|
53
|
+
## The chain
|
|
54
|
+
|
|
55
|
+
A number in a manuscript names a claim. The claim names a run. The run names its outputs. The
|
|
56
|
+
outputs were hashed when they were recorded. The inputs were hashed before the run started.
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
manuscript → claim → run → output file → sha256
|
|
60
|
+
input files → sha256
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`results verify --files` walks the whole thing and tells you what moved.
|
|
64
|
+
|
|
65
|
+
## Data-access levels
|
|
66
|
+
|
|
67
|
+
The access timeline is what makes the confirmatory/exploratory distinction verifiable.
|
|
68
|
+
|
|
69
|
+
| Level | Meaning |
|
|
70
|
+
|-------|---------|
|
|
71
|
+
| `nothing seen` | No target data touched |
|
|
72
|
+
| `metadata only` | Structure, region names, sample sizes — not outcomes |
|
|
73
|
+
| `structure seen` | Data shape and distributions, not the target variable |
|
|
74
|
+
| `outcomes seen` | The dependent variable was observed |
|
|
75
|
+
|
|
76
|
+
An analysis registered after `outcomes seen` is retrospective.
|
|
77
|
+
|
|
78
|
+
## Verify output
|
|
79
|
+
|
|
80
|
+
| Result | Meaning |
|
|
81
|
+
|--------|---------|
|
|
82
|
+
| `chain intact` | Every event's prev_hash matches the line before it |
|
|
83
|
+
| `CHAIN BROKEN` | The ledger was edited after it was written |
|
|
84
|
+
| `ok` | File matches its recorded hash |
|
|
85
|
+
| `CHANGED` | File was modified since it was recorded |
|
|
86
|
+
| `MISSING` | File no longer exists |
|
|
87
|
+
|
|
88
|
+
## The ledger
|
|
89
|
+
|
|
90
|
+
Append-only JSONL in `.results/ledger.jsonl`. Each line is hash-chained to the previous — editing
|
|
91
|
+
or inserting a line breaks the chain. `git diff` shows what changed; `results verify` checks
|
|
92
|
+
whether it should have.
|
|
93
|
+
|
|
94
|
+
## Claude Code
|
|
95
|
+
|
|
96
|
+
`plugin/` is a Claude Code plugin that tells Claude when to reach for the CLI.
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
/plugin marketplace add elliottower/results
|
|
100
|
+
/plugin install results@results
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
MIT licensed.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "results",
|
|
3
|
+
"description": "Seal a run's inputs, record what it produced, bind claims to outputs, and verify the chain",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"author": { "name": "Elliot Tower", "email": "elliot@elliottower.ai" },
|
|
6
|
+
"repository": "https://github.com/elliottower/results",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"keywords": ["results", "provenance", "reproducibility", "open science"]
|
|
9
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# The Claude Code plugin
|
|
2
|
+
|
|
3
|
+
Tells Claude the `results` CLI exists and when to reach for it. The CLI does the work; this
|
|
4
|
+
adds nothing the CLI cannot do.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
/plugin marketplace add elliottower/results
|
|
10
|
+
/plugin install results@results
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The plugin ships instructions, not binaries. Install the tool too:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
uv tool install results-cli
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
For development against a checkout:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
/plugin marketplace add ~/Documents/GitHub/results
|
|
23
|
+
/plugin install results@results
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## What it changes
|
|
27
|
+
|
|
28
|
+
Claude will seal inputs before running a computation rather than after, record outputs with their
|
|
29
|
+
hashes, and bind manuscript claims to specific runs instead of leaving numbers untraced. It will
|
|
30
|
+
also record data-access events so the distinction between confirmatory and exploratory analysis
|
|
31
|
+
is verifiable from the timeline, not from a self-report.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: results
|
|
3
|
+
description: Seal a run's inputs before computing, record outputs after, bind manuscript claims to specific runs, and verify the hash chain. Use before launching any computation whose output will appear in a paper, after a run completes, when a number in a manuscript needs tracing to its source, or when asked whether results are still what they were. Requires the `results` CLI (`uv tool install results-cli`).
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# results
|
|
7
|
+
|
|
8
|
+
Seal a run, record what it produced, and verify the chain.
|
|
9
|
+
|
|
10
|
+
## The rule that matters
|
|
11
|
+
|
|
12
|
+
**Seal before you run. Claim after you verify.**
|
|
13
|
+
|
|
14
|
+
A number in a manuscript is trustworthy when it names the run that produced it, the run names the
|
|
15
|
+
inputs that were sealed before it started, and the chain from input to claim has not been broken.
|
|
16
|
+
Every way this fails is a version of the result moving without a trace — a script edited after
|
|
17
|
+
the seal, an output overwritten between the run and the claim, a claim attached to a run that
|
|
18
|
+
no longer exists.
|
|
19
|
+
|
|
20
|
+
## Commands
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
results init # start tracking results here
|
|
24
|
+
results seal <file>... [--role input] # hash inputs before a run
|
|
25
|
+
results access <note> [--level ...] # record a data-access event
|
|
26
|
+
results run <file>... --run-id <id> # record outputs after a run
|
|
27
|
+
results claim <text> --run-id <id> # bind a manuscript claim to a run
|
|
28
|
+
results verify [--files] # check the chain and every hash it names
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## The workflow, in order
|
|
32
|
+
|
|
33
|
+
1. **`results init`** in the experiment directory.
|
|
34
|
+
2. **`results seal prereg.md script.py data.csv`** — hash every input before computing.
|
|
35
|
+
3. **`results access "downloaded zenodo metadata" --level "metadata only"`** — record what
|
|
36
|
+
was seen and when. This is the data-access taint the chain uses to distinguish confirmatory
|
|
37
|
+
from exploratory.
|
|
38
|
+
4. **Run the computation.**
|
|
39
|
+
5. **`results run output.json --run-id exp_001`** — hash the outputs.
|
|
40
|
+
6. **`results claim "ICC = 0.42" --run-id exp_001 --confirmatory --location "Table 2"`** —
|
|
41
|
+
bind the manuscript claim to the run.
|
|
42
|
+
7. **`results verify --files`** — check everything.
|
|
43
|
+
|
|
44
|
+
## Data-access levels
|
|
45
|
+
|
|
46
|
+
Four levels, in order of exposure:
|
|
47
|
+
|
|
48
|
+
| level | meaning |
|
|
49
|
+
|---|---|
|
|
50
|
+
| `nothing seen` | no target data touched |
|
|
51
|
+
| `metadata only` | structure, region names, sample sizes — not outcomes |
|
|
52
|
+
| `structure seen` | data shape and distributions, but not the target variable |
|
|
53
|
+
| `outcomes seen` | the dependent variable was observed |
|
|
54
|
+
|
|
55
|
+
An analysis registered after `outcomes seen` is retrospective, not confirmatory. The level is
|
|
56
|
+
recorded, not judged — log the honest one even when it is the damaging one.
|
|
57
|
+
|
|
58
|
+
## Reading `results verify`
|
|
59
|
+
|
|
60
|
+
The chain check is pass/fail:
|
|
61
|
+
|
|
62
|
+
| | |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `chain intact` | every event's prev_hash matches the hash of the line before it |
|
|
65
|
+
| `CHAIN BROKEN` | a line was edited, inserted, or deleted after it was written |
|
|
66
|
+
|
|
67
|
+
With `--files`, every sealed input and recorded output is re-hashed against its current state:
|
|
68
|
+
|
|
69
|
+
| | |
|
|
70
|
+
|---|---|
|
|
71
|
+
| `ok` | file matches its recorded hash |
|
|
72
|
+
| `CHANGED` | file was modified since it was sealed or recorded |
|
|
73
|
+
| `MISSING` | file no longer exists at the recorded path |
|
|
74
|
+
|
|
75
|
+
**`CHAIN BROKEN` means the ledger was tampered with.** It does not mean the results are wrong —
|
|
76
|
+
it means the evidence that they are right was damaged. Restore from git history.
|
|
77
|
+
|
|
78
|
+
## Claims
|
|
79
|
+
|
|
80
|
+
A claim names a run. A run names its outputs. The outputs were hashed when they were recorded.
|
|
81
|
+
This is the whole chain: manuscript → claim → run → output file → hash. `verify` walks it.
|
|
82
|
+
|
|
83
|
+
`--confirmatory` marks the claim as backed by a pre-registered hypothesis. Without it, the claim
|
|
84
|
+
is exploratory. The distinction is recorded, not enforced — the access timeline is what makes it
|
|
85
|
+
verifiable.
|
|
86
|
+
|
|
87
|
+
## When to reach for this
|
|
88
|
+
|
|
89
|
+
- Before launching any computation whose output will be reported
|
|
90
|
+
- After a run completes, to record what it produced
|
|
91
|
+
- When a number in a manuscript needs tracing to its source run
|
|
92
|
+
- When asked whether results are still what they were
|
|
93
|
+
- In CI, as `results verify --files`
|
|
94
|
+
|
|
95
|
+
## What it will not do
|
|
96
|
+
|
|
97
|
+
It cannot prove the computation was correct. A sealed script that produces the wrong answer
|
|
98
|
+
passes every check here. It also cannot prove the access timeline is honest — the events are
|
|
99
|
+
self-reported, and the tool records them without questioning. The chain is evidence, not proof.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "results-cli"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Seal a run, record what it produced, and verify the chain"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [{ name = "Elliot Tower", email = "elliot@elliottower.ai" }]
|
|
9
|
+
keywords = ["reproducibility", "provenance", "open science", "results"]
|
|
10
|
+
dependencies = []
|
|
11
|
+
|
|
12
|
+
[project.scripts]
|
|
13
|
+
results = "results.cli:main"
|
|
14
|
+
|
|
15
|
+
[build-system]
|
|
16
|
+
requires = ["hatchling"]
|
|
17
|
+
build-backend = "hatchling.build"
|
|
18
|
+
|
|
19
|
+
[tool.hatch.build.targets.wheel]
|
|
20
|
+
packages = ["src/results"]
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Seal a run, record what it produced, and verify the chain.
|
|
2
|
+
|
|
3
|
+
results init start tracking results here
|
|
4
|
+
results seal <file>... hash inputs before a run (prereg, script, data)
|
|
5
|
+
results access <note> record a data-access event (what you looked at, when)
|
|
6
|
+
results run <file>... record outputs after a run completes
|
|
7
|
+
results claim <text> bind a manuscript claim to a run's output
|
|
8
|
+
results verify check the ledger chain and every hash it names
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import os
|
|
14
|
+
import pathlib
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from results import ledger
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
RESULTS_DIR = ".results"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def find_root(start: pathlib.Path | None = None) -> pathlib.Path | None:
|
|
24
|
+
here = (start or pathlib.Path.cwd()).resolve()
|
|
25
|
+
for d in [here, *here.parents]:
|
|
26
|
+
if (d / RESULTS_DIR).is_dir():
|
|
27
|
+
return d / RESULTS_DIR
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def require_root() -> pathlib.Path:
|
|
32
|
+
root = find_root()
|
|
33
|
+
if root is None:
|
|
34
|
+
print(f"no {RESULTS_DIR}/ here or above. `results init` makes one.")
|
|
35
|
+
sys.exit(2)
|
|
36
|
+
return root
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def ledger_path(root: pathlib.Path) -> pathlib.Path:
|
|
40
|
+
return root / ledger.LEDGER
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def cmd_init(a) -> int:
|
|
44
|
+
d = pathlib.Path.cwd() / RESULTS_DIR
|
|
45
|
+
if d.exists():
|
|
46
|
+
print(f"{d} already exists.")
|
|
47
|
+
return 1
|
|
48
|
+
d.mkdir()
|
|
49
|
+
lp = d / ledger.LEDGER
|
|
50
|
+
lp.touch()
|
|
51
|
+
ledger.append_event(lp, {"event": "init"})
|
|
52
|
+
print(f"created {RESULTS_DIR}/")
|
|
53
|
+
print(f" {ledger.LEDGER} append-only event log")
|
|
54
|
+
print("\nseal your inputs before running: `results seal prereg.md script.py data.csv`")
|
|
55
|
+
return 0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def cmd_seal(a) -> int:
|
|
59
|
+
root = require_root()
|
|
60
|
+
lp = ledger_path(root)
|
|
61
|
+
sealed = []
|
|
62
|
+
for name in a.files:
|
|
63
|
+
p = pathlib.Path(name).resolve()
|
|
64
|
+
if not p.is_file():
|
|
65
|
+
print(f"not a file: {name}")
|
|
66
|
+
return 1
|
|
67
|
+
digest = ledger.sha256_of_file(p)
|
|
68
|
+
sealed.append({"path": os.path.relpath(p), "sha256": digest})
|
|
69
|
+
ev = ledger.append_event(lp, {
|
|
70
|
+
"event": "seal",
|
|
71
|
+
"role": a.role,
|
|
72
|
+
"files": sealed,
|
|
73
|
+
})
|
|
74
|
+
print(f"sealed {len(sealed)} file(s) as {a.role}")
|
|
75
|
+
for s in sealed:
|
|
76
|
+
print(f" {s['sha256'][:16]}… {s['path']}")
|
|
77
|
+
return 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def cmd_access(a) -> int:
|
|
81
|
+
root = require_root()
|
|
82
|
+
lp = ledger_path(root)
|
|
83
|
+
if a.level not in ACCESS_LEVELS:
|
|
84
|
+
print(f"level must be one of: {', '.join(ACCESS_LEVELS)}")
|
|
85
|
+
return 1
|
|
86
|
+
ev = ledger.append_event(lp, {
|
|
87
|
+
"event": "access",
|
|
88
|
+
"level": a.level,
|
|
89
|
+
"note": a.note,
|
|
90
|
+
})
|
|
91
|
+
print(f"recorded: {a.level} — {a.note}")
|
|
92
|
+
if a.level == "outcomes seen":
|
|
93
|
+
print("\nany analysis registered after this is retrospective, not confirmatory.")
|
|
94
|
+
return 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def cmd_run(a) -> int:
|
|
98
|
+
root = require_root()
|
|
99
|
+
lp = ledger_path(root)
|
|
100
|
+
existing = ledger.read_ledger(lp)
|
|
101
|
+
existing_ids = {e["run_id"] for e in existing if e.get("event") == "run"}
|
|
102
|
+
if a.run_id in existing_ids:
|
|
103
|
+
print(f"warning: run id '{a.run_id}' already exists in the ledger.")
|
|
104
|
+
print("the new run will be recorded alongside the old one.")
|
|
105
|
+
outputs = []
|
|
106
|
+
for name in a.files:
|
|
107
|
+
p = pathlib.Path(name).resolve()
|
|
108
|
+
if not p.is_file():
|
|
109
|
+
print(f"not a file: {name}")
|
|
110
|
+
return 1
|
|
111
|
+
digest = ledger.sha256_of_file(p)
|
|
112
|
+
outputs.append({"path": os.path.relpath(p), "sha256": digest})
|
|
113
|
+
ev = ledger.append_event(lp, {
|
|
114
|
+
"event": "run",
|
|
115
|
+
"run_id": a.run_id,
|
|
116
|
+
"outputs": outputs,
|
|
117
|
+
"note": a.note or "",
|
|
118
|
+
})
|
|
119
|
+
print(f"run {a.run_id}: {len(outputs)} output(s)")
|
|
120
|
+
for o in outputs:
|
|
121
|
+
print(f" {o['sha256'][:16]}… {o['path']}")
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def cmd_claim(a) -> int:
|
|
126
|
+
root = require_root()
|
|
127
|
+
lp = ledger_path(root)
|
|
128
|
+
|
|
129
|
+
events = ledger.read_ledger(lp)
|
|
130
|
+
run_ids = {e["run_id"] for e in events if e.get("event") == "run"}
|
|
131
|
+
if a.run_id not in run_ids:
|
|
132
|
+
print(f"no run with id '{a.run_id}' in the ledger.")
|
|
133
|
+
print(f"known runs: {', '.join(sorted(run_ids)) or '(none)'}")
|
|
134
|
+
return 1
|
|
135
|
+
|
|
136
|
+
ev = ledger.append_event(lp, {
|
|
137
|
+
"event": "claim",
|
|
138
|
+
"claim": a.text,
|
|
139
|
+
"run_id": a.run_id,
|
|
140
|
+
"confirmatory": a.confirmatory,
|
|
141
|
+
"location": a.location or "",
|
|
142
|
+
})
|
|
143
|
+
status = "confirmatory" if a.confirmatory else "exploratory"
|
|
144
|
+
print(f"claim ({status}): {a.text[:72]}")
|
|
145
|
+
print(f" backed by run: {a.run_id}")
|
|
146
|
+
if a.location:
|
|
147
|
+
print(f" appears in: {a.location}")
|
|
148
|
+
return 0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def cmd_verify(a) -> int:
|
|
152
|
+
root = require_root()
|
|
153
|
+
lp = ledger_path(root)
|
|
154
|
+
|
|
155
|
+
ok, problems = ledger.verify_chain(lp)
|
|
156
|
+
if not ok:
|
|
157
|
+
print("CHAIN BROKEN")
|
|
158
|
+
for p in problems:
|
|
159
|
+
print(f" {p}")
|
|
160
|
+
return 1
|
|
161
|
+
|
|
162
|
+
events = ledger.read_ledger(lp)
|
|
163
|
+
print(f"chain intact: {len(events)} events\n")
|
|
164
|
+
|
|
165
|
+
counts = {}
|
|
166
|
+
for e in events:
|
|
167
|
+
t = e.get("event", "?")
|
|
168
|
+
counts[t] = counts.get(t, 0) + 1
|
|
169
|
+
for t, n in sorted(counts.items()):
|
|
170
|
+
print(f" {t:<12}{n:>5}")
|
|
171
|
+
|
|
172
|
+
drift = 0
|
|
173
|
+
if a.files:
|
|
174
|
+
print("\nfile hashes:")
|
|
175
|
+
file_hashes = {}
|
|
176
|
+
for e in events:
|
|
177
|
+
for f in e.get("files", []) + e.get("outputs", []):
|
|
178
|
+
file_hashes[f["path"]] = f["sha256"]
|
|
179
|
+
for path, expected in sorted(file_hashes.items()):
|
|
180
|
+
p = pathlib.Path(path)
|
|
181
|
+
if not p.exists():
|
|
182
|
+
print(f" MISSING {path}")
|
|
183
|
+
drift += 1
|
|
184
|
+
else:
|
|
185
|
+
actual = ledger.sha256_of_file(p)
|
|
186
|
+
if actual == expected:
|
|
187
|
+
print(f" ok {path}")
|
|
188
|
+
else:
|
|
189
|
+
print(f" CHANGED {path}")
|
|
190
|
+
print(f" sealed {expected[:16]}…")
|
|
191
|
+
print(f" now {actual[:16]}…")
|
|
192
|
+
drift += 1
|
|
193
|
+
if drift:
|
|
194
|
+
print(f"\n{drift} file(s) changed or missing since they were recorded.")
|
|
195
|
+
return 1
|
|
196
|
+
|
|
197
|
+
access_events = [e for e in events if e.get("event") == "access"]
|
|
198
|
+
if access_events:
|
|
199
|
+
print("\ndata access timeline:")
|
|
200
|
+
for e in access_events:
|
|
201
|
+
print(f" {e['timestamp'][:19]} {e['level']:<20} {e.get('note', '')}")
|
|
202
|
+
|
|
203
|
+
claims = [e for e in events if e.get("event") == "claim"]
|
|
204
|
+
if claims:
|
|
205
|
+
unlinked = []
|
|
206
|
+
for c in claims:
|
|
207
|
+
run_events = [e for e in events
|
|
208
|
+
if e.get("event") == "run" and e.get("run_id") == c.get("run_id")]
|
|
209
|
+
if not run_events:
|
|
210
|
+
unlinked.append(c)
|
|
211
|
+
if unlinked:
|
|
212
|
+
print(f"\n{len(unlinked)} claim(s) reference missing runs:")
|
|
213
|
+
for c in unlinked:
|
|
214
|
+
print(f" {c['claim'][:60]} (run: {c.get('run_id')})")
|
|
215
|
+
|
|
216
|
+
print()
|
|
217
|
+
if ok and not (a.files and drift):
|
|
218
|
+
print("all checks passed.")
|
|
219
|
+
return 0
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
ACCESS_LEVELS = [
|
|
223
|
+
"nothing seen",
|
|
224
|
+
"metadata only",
|
|
225
|
+
"structure seen",
|
|
226
|
+
"outcomes seen",
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def main() -> int:
|
|
231
|
+
ap = argparse.ArgumentParser(prog="results", description=__doc__.split("\n")[0])
|
|
232
|
+
sub = ap.add_subparsers(dest="cmd")
|
|
233
|
+
|
|
234
|
+
sub.add_parser("init", help="start tracking results here")
|
|
235
|
+
|
|
236
|
+
s = sub.add_parser("seal", help="hash inputs before a run")
|
|
237
|
+
s.add_argument("files", nargs="+")
|
|
238
|
+
s.add_argument("--role", default="input",
|
|
239
|
+
help="what these files are: input, prereg, script, data")
|
|
240
|
+
s.set_defaults(fn=cmd_seal)
|
|
241
|
+
|
|
242
|
+
ac = sub.add_parser("access", help="record a data-access event")
|
|
243
|
+
ac.add_argument("note", help="what was accessed and why")
|
|
244
|
+
ac.add_argument("--level", default="metadata only",
|
|
245
|
+
help=f"one of: {', '.join(ACCESS_LEVELS)}")
|
|
246
|
+
ac.set_defaults(fn=cmd_access)
|
|
247
|
+
|
|
248
|
+
r = sub.add_parser("run", help="record outputs after a run")
|
|
249
|
+
r.add_argument("files", nargs="+")
|
|
250
|
+
r.add_argument("--run-id", required=True, help="a name for this run")
|
|
251
|
+
r.add_argument("--note", help="what this run computed")
|
|
252
|
+
r.set_defaults(fn=cmd_run)
|
|
253
|
+
|
|
254
|
+
cl = sub.add_parser("claim", help="bind a manuscript claim to a run")
|
|
255
|
+
cl.add_argument("text", help="the claim, as it appears in the manuscript")
|
|
256
|
+
cl.add_argument("--run-id", required=True, help="which run backs this claim")
|
|
257
|
+
cl.add_argument("--confirmatory", action="store_true",
|
|
258
|
+
help="mark as confirmatory (default: exploratory)")
|
|
259
|
+
cl.add_argument("--location", help="where in the manuscript: Table 2, Section 4.1, etc.")
|
|
260
|
+
cl.set_defaults(fn=cmd_claim)
|
|
261
|
+
|
|
262
|
+
v = sub.add_parser("verify", help="check the ledger and every hash it names")
|
|
263
|
+
v.add_argument("--files", action="store_true",
|
|
264
|
+
help="also check that sealed/output files still match their hashes")
|
|
265
|
+
v.set_defaults(fn=cmd_verify)
|
|
266
|
+
|
|
267
|
+
a = ap.parse_args()
|
|
268
|
+
if not a.cmd:
|
|
269
|
+
ap.print_help()
|
|
270
|
+
return 0
|
|
271
|
+
if a.cmd == "init":
|
|
272
|
+
return cmd_init(a)
|
|
273
|
+
return a.fn(a)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
if __name__ == "__main__":
|
|
277
|
+
sys.exit(main())
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Append-only JSONL ledger: every event is one line, hash-chained to the previous."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import datetime
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import pathlib
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
LEDGER = "ledger.jsonl"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sha256_of_file(path: pathlib.Path) -> str:
|
|
14
|
+
h = hashlib.sha256()
|
|
15
|
+
with open(path, "rb") as f:
|
|
16
|
+
for chunk in iter(lambda: f.read(1 << 16), b""):
|
|
17
|
+
h.update(chunk)
|
|
18
|
+
return h.hexdigest()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def sha256_of_str(s: str) -> str:
|
|
22
|
+
return hashlib.sha256(s.encode()).hexdigest()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def now_iso() -> str:
|
|
26
|
+
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def last_hash(ledger: pathlib.Path) -> str:
|
|
30
|
+
"""The hash of the last line as stored, or a zero hash if the ledger is empty."""
|
|
31
|
+
if not ledger.exists() or ledger.stat().st_size == 0:
|
|
32
|
+
return "0" * 64
|
|
33
|
+
with open(ledger, "rb") as f:
|
|
34
|
+
last = b""
|
|
35
|
+
for line in f:
|
|
36
|
+
if line.strip():
|
|
37
|
+
last = line
|
|
38
|
+
return sha256_of_str(last.decode().strip())
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def append_event(ledger: pathlib.Path, event: dict) -> dict:
|
|
42
|
+
"""Write one event to the ledger. Returns the event with chain fields added."""
|
|
43
|
+
event["timestamp"] = now_iso()
|
|
44
|
+
event["prev_hash"] = last_hash(ledger)
|
|
45
|
+
line = json.dumps(event, separators=(",", ":"), sort_keys=True)
|
|
46
|
+
with open(ledger, "a") as f:
|
|
47
|
+
f.write(line + "\n")
|
|
48
|
+
return event
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_ledger(ledger: pathlib.Path) -> list[dict]:
|
|
52
|
+
if not ledger.exists():
|
|
53
|
+
return []
|
|
54
|
+
events = []
|
|
55
|
+
for line in ledger.read_text().splitlines():
|
|
56
|
+
line = line.strip()
|
|
57
|
+
if line:
|
|
58
|
+
events.append(json.loads(line))
|
|
59
|
+
return events
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def verify_chain(ledger: pathlib.Path) -> tuple[bool, list[str]]:
|
|
63
|
+
"""Check that every line's prev_hash matches the hash of the previous line as stored."""
|
|
64
|
+
if not ledger.exists():
|
|
65
|
+
return True, []
|
|
66
|
+
lines = [ln.strip() for ln in ledger.read_text().splitlines() if ln.strip()]
|
|
67
|
+
if not lines:
|
|
68
|
+
return True, []
|
|
69
|
+
problems = []
|
|
70
|
+
prev = "0" * 64
|
|
71
|
+
for i, raw in enumerate(lines):
|
|
72
|
+
ev = json.loads(raw)
|
|
73
|
+
if ev.get("prev_hash") != prev:
|
|
74
|
+
problems.append(
|
|
75
|
+
f"line {i + 1}: prev_hash mismatch — expected {prev[:16]}…, "
|
|
76
|
+
f"got {ev.get('prev_hash', '???')[:16]}…")
|
|
77
|
+
prev = sha256_of_str(raw)
|
|
78
|
+
return len(problems) == 0, problems
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Tests for the results CLI commands."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from results import ledger
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def run_cli(*args, cwd=None):
|
|
11
|
+
return subprocess.run(
|
|
12
|
+
[sys.executable, "-m", "results.cli", *args],
|
|
13
|
+
cwd=cwd, capture_output=True, text=True,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_init_creates_results_dir(tmp_path):
|
|
18
|
+
r = run_cli("init", cwd=tmp_path)
|
|
19
|
+
assert r.returncode == 0
|
|
20
|
+
assert (tmp_path / ".results" / "ledger.jsonl").exists()
|
|
21
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
22
|
+
assert len(events) == 1
|
|
23
|
+
assert events[0]["event"] == "init"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_init_twice_fails(tmp_path):
|
|
27
|
+
run_cli("init", cwd=tmp_path)
|
|
28
|
+
r = run_cli("init", cwd=tmp_path)
|
|
29
|
+
assert r.returncode == 1
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_seal_records_file_hashes(tmp_path):
|
|
33
|
+
run_cli("init", cwd=tmp_path)
|
|
34
|
+
(tmp_path / "script.py").write_text("print('hello')\n")
|
|
35
|
+
(tmp_path / "data.csv").write_text("a,b\n1,2\n")
|
|
36
|
+
r = run_cli("seal", "script.py", "data.csv", "--role", "input", cwd=tmp_path)
|
|
37
|
+
assert r.returncode == 0
|
|
38
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
39
|
+
seal = [e for e in events if e["event"] == "seal"]
|
|
40
|
+
assert len(seal) == 1
|
|
41
|
+
assert len(seal[0]["files"]) == 2
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_access_records_event(tmp_path):
|
|
45
|
+
run_cli("init", cwd=tmp_path)
|
|
46
|
+
r = run_cli("access", "read zenodo metadata", "--level", "metadata only", cwd=tmp_path)
|
|
47
|
+
assert r.returncode == 0
|
|
48
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
49
|
+
access = [e for e in events if e["event"] == "access"]
|
|
50
|
+
assert len(access) == 1
|
|
51
|
+
assert access[0]["level"] == "metadata only"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_access_rejects_invalid_level(tmp_path):
|
|
55
|
+
run_cli("init", cwd=tmp_path)
|
|
56
|
+
r = run_cli("access", "whatever", "--level", "banana", cwd=tmp_path)
|
|
57
|
+
assert r.returncode == 1
|
|
58
|
+
assert "level must be" in r.stdout
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_run_records_outputs(tmp_path):
|
|
62
|
+
run_cli("init", cwd=tmp_path)
|
|
63
|
+
(tmp_path / "results.json").write_text('{"icc": 0.42}\n')
|
|
64
|
+
r = run_cli("run", "results.json", "--run-id", "run_001", cwd=tmp_path)
|
|
65
|
+
assert r.returncode == 0
|
|
66
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
67
|
+
runs = [e for e in events if e["event"] == "run"]
|
|
68
|
+
assert len(runs) == 1
|
|
69
|
+
assert runs[0]["run_id"] == "run_001"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_run_warns_on_duplicate_id(tmp_path):
|
|
73
|
+
run_cli("init", cwd=tmp_path)
|
|
74
|
+
(tmp_path / "out1.json").write_text('{"a": 1}\n')
|
|
75
|
+
(tmp_path / "out2.json").write_text('{"a": 2}\n')
|
|
76
|
+
run_cli("run", "out1.json", "--run-id", "exp_001", cwd=tmp_path)
|
|
77
|
+
r = run_cli("run", "out2.json", "--run-id", "exp_001", cwd=tmp_path)
|
|
78
|
+
assert r.returncode == 0
|
|
79
|
+
assert "already exists" in r.stdout
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_claim_requires_existing_run(tmp_path):
|
|
83
|
+
run_cli("init", cwd=tmp_path)
|
|
84
|
+
r = run_cli("claim", "ICC = 0.42", "--run-id", "run_001", cwd=tmp_path)
|
|
85
|
+
assert r.returncode == 1
|
|
86
|
+
assert "no run" in r.stdout
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_claim_succeeds_after_run(tmp_path):
|
|
90
|
+
run_cli("init", cwd=tmp_path)
|
|
91
|
+
(tmp_path / "results.json").write_text('{"icc": 0.42}\n')
|
|
92
|
+
run_cli("run", "results.json", "--run-id", "run_001", cwd=tmp_path)
|
|
93
|
+
r = run_cli("claim", "ICC = 0.42", "--run-id", "run_001",
|
|
94
|
+
"--confirmatory", "--location", "Table 2", cwd=tmp_path)
|
|
95
|
+
assert r.returncode == 0
|
|
96
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
97
|
+
claims = [e for e in events if e["event"] == "claim"]
|
|
98
|
+
assert len(claims) == 1
|
|
99
|
+
assert claims[0]["confirmatory"] is True
|
|
100
|
+
assert claims[0]["location"] == "Table 2"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_verify_passes_on_clean_ledger(tmp_path):
|
|
104
|
+
run_cli("init", cwd=tmp_path)
|
|
105
|
+
r = run_cli("verify", cwd=tmp_path)
|
|
106
|
+
assert r.returncode == 0
|
|
107
|
+
assert "chain intact" in r.stdout
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_verify_files_catches_drift(tmp_path):
|
|
111
|
+
run_cli("init", cwd=tmp_path)
|
|
112
|
+
(tmp_path / "data.csv").write_text("a,b\n1,2\n")
|
|
113
|
+
run_cli("seal", "data.csv", cwd=tmp_path)
|
|
114
|
+
(tmp_path / "data.csv").write_text("a,b\n1,2\n3,4\n")
|
|
115
|
+
r = run_cli("verify", "--files", cwd=tmp_path)
|
|
116
|
+
assert r.returncode == 1
|
|
117
|
+
assert "CHANGED" in r.stdout
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_full_workflow(tmp_path):
|
|
121
|
+
run_cli("init", cwd=tmp_path)
|
|
122
|
+
(tmp_path / "prereg.md").write_text("# My prereg\n")
|
|
123
|
+
(tmp_path / "script.py").write_text("import json\n")
|
|
124
|
+
run_cli("seal", "prereg.md", "script.py", "--role", "prereg", cwd=tmp_path)
|
|
125
|
+
run_cli("access", "downloaded zenodo metadata", "--level", "metadata only", cwd=tmp_path)
|
|
126
|
+
(tmp_path / "out.json").write_text('{"result": 42}\n')
|
|
127
|
+
run_cli("run", "out.json", "--run-id", "exp_001", "--note", "first analysis", cwd=tmp_path)
|
|
128
|
+
run_cli("claim", "the answer is 42", "--run-id", "exp_001",
|
|
129
|
+
"--confirmatory", "--location", "Section 3", cwd=tmp_path)
|
|
130
|
+
r = run_cli("verify", "--files", cwd=tmp_path)
|
|
131
|
+
assert r.returncode == 0
|
|
132
|
+
assert "chain intact" in r.stdout
|
|
133
|
+
assert "all checks passed" in r.stdout
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Tests for the append-only hash-chained ledger."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
from results import ledger
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_empty_ledger_has_zero_hash(tmp_path):
|
|
10
|
+
lp = tmp_path / "ledger.jsonl"
|
|
11
|
+
lp.touch()
|
|
12
|
+
assert ledger.last_hash(lp) == "0" * 64
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_append_and_read_round_trips(tmp_path):
|
|
16
|
+
lp = tmp_path / "ledger.jsonl"
|
|
17
|
+
lp.touch()
|
|
18
|
+
ledger.append_event(lp, {"event": "init"})
|
|
19
|
+
ledger.append_event(lp, {"event": "seal", "files": [{"path": "x.py", "sha256": "abc"}]})
|
|
20
|
+
events = ledger.read_ledger(lp)
|
|
21
|
+
assert len(events) == 2
|
|
22
|
+
assert events[0]["event"] == "init"
|
|
23
|
+
assert events[1]["event"] == "seal"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_chain_verification_passes_on_clean_ledger(tmp_path):
|
|
27
|
+
lp = tmp_path / "ledger.jsonl"
|
|
28
|
+
lp.touch()
|
|
29
|
+
for i in range(5):
|
|
30
|
+
ledger.append_event(lp, {"event": "test", "i": i})
|
|
31
|
+
ok, problems = ledger.verify_chain(lp)
|
|
32
|
+
assert ok
|
|
33
|
+
assert problems == []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_chain_verification_catches_tampering(tmp_path):
|
|
37
|
+
lp = tmp_path / "ledger.jsonl"
|
|
38
|
+
lp.touch()
|
|
39
|
+
ledger.append_event(lp, {"event": "init"})
|
|
40
|
+
ledger.append_event(lp, {"event": "seal"})
|
|
41
|
+
ledger.append_event(lp, {"event": "run"})
|
|
42
|
+
|
|
43
|
+
lines = lp.read_text().splitlines()
|
|
44
|
+
tampered = json.loads(lines[1])
|
|
45
|
+
tampered["event"] = "TAMPERED"
|
|
46
|
+
lines[1] = json.dumps(tampered, separators=(",", ":"), sort_keys=True)
|
|
47
|
+
lp.write_text("\n".join(lines) + "\n")
|
|
48
|
+
|
|
49
|
+
ok, problems = ledger.verify_chain(lp)
|
|
50
|
+
assert not ok
|
|
51
|
+
assert len(problems) >= 1
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_sha256_of_file_is_deterministic(tmp_path):
|
|
55
|
+
f = tmp_path / "data.txt"
|
|
56
|
+
f.write_text("hello world\n")
|
|
57
|
+
h1 = ledger.sha256_of_file(f)
|
|
58
|
+
h2 = ledger.sha256_of_file(f)
|
|
59
|
+
assert h1 == h2
|
|
60
|
+
assert len(h1) == 64
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_first_event_prev_hash_is_zero(tmp_path):
|
|
64
|
+
lp = tmp_path / "ledger.jsonl"
|
|
65
|
+
lp.touch()
|
|
66
|
+
ev = ledger.append_event(lp, {"event": "init"})
|
|
67
|
+
assert ev["prev_hash"] == "0" * 64
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_second_event_prev_hash_chains_to_first(tmp_path):
|
|
71
|
+
lp = tmp_path / "ledger.jsonl"
|
|
72
|
+
lp.touch()
|
|
73
|
+
ledger.append_event(lp, {"event": "first"})
|
|
74
|
+
ev2 = ledger.append_event(lp, {"event": "second"})
|
|
75
|
+
first_line = lp.read_text().splitlines()[0].strip()
|
|
76
|
+
assert ev2["prev_hash"] == ledger.sha256_of_str(first_line)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_verify_catches_inserted_line(tmp_path):
|
|
80
|
+
lp = tmp_path / "ledger.jsonl"
|
|
81
|
+
lp.touch()
|
|
82
|
+
ledger.append_event(lp, {"event": "first"})
|
|
83
|
+
ledger.append_event(lp, {"event": "second"})
|
|
84
|
+
lines = lp.read_text().splitlines()
|
|
85
|
+
injected = '{"event":"injected","prev_hash":"fake","timestamp":"2026-01-01T00:00:00+00:00"}'
|
|
86
|
+
lines.insert(1, injected)
|
|
87
|
+
lp.write_text("\n".join(lines) + "\n")
|
|
88
|
+
ok, problems = ledger.verify_chain(lp)
|
|
89
|
+
assert not ok
|