results-cli 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {results_cli-0.2.0 → results_cli-0.3.0}/PKG-INFO +37 -7
- {results_cli-0.2.0 → results_cli-0.3.0}/README.md +31 -6
- results_cli-0.3.0/plugin/.claude-plugin/plugin.json +19 -0
- results_cli-0.3.0/plugin/README.md +50 -0
- results_cli-0.3.0/plugin/commands/results-check.md +25 -0
- results_cli-0.3.0/plugin/hooks/hooks.json +13 -0
- results_cli-0.3.0/plugin/hooks/unbound_numbers.py +187 -0
- results_cli-0.3.0/plugin/skills/results/SKILL.md +181 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/pyproject.toml +6 -2
- results_cli-0.3.0/src/results/audit.py +256 -0
- results_cli-0.3.0/src/results/cli.py +165 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/src/results/ledger.py +9 -11
- results_cli-0.3.0/src/results/manuscript.py +245 -0
- results_cli-0.3.0/src/results/paths.py +53 -0
- results_cli-0.3.0/src/results/record.py +201 -0
- results_cli-0.3.0/src/results/timeline.py +83 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/tests/test_cli.py +82 -0
- results_cli-0.3.0/tests/test_coverage_and_hook.py +244 -0
- results_cli-0.3.0/tests/test_manuscript.py +176 -0
- results_cli-0.3.0/tests/test_timeline.py +215 -0
- results_cli-0.2.0/.claude-plugin/marketplace.json +0 -17
- results_cli-0.2.0/plugin/.claude-plugin/plugin.json +0 -9
- results_cli-0.2.0/plugin/README.md +0 -31
- results_cli-0.2.0/plugin/skills/results/SKILL.md +0 -99
- results_cli-0.2.0/src/results/cli.py +0 -464
- {results_cli-0.2.0 → results_cli-0.3.0}/.gitignore +0 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/LICENSE +0 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/src/results/__init__.py +0 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/tests/test_ledger.py +0 -0
- {results_cli-0.2.0 → results_cli-0.3.0}/tests/test_tamper.py +0 -0
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: results-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Seal a run, record what it produced, and verify the chain
|
|
5
|
+
Project-URL: Changelog, https://github.com/elliottower/reproducible-science/blob/main/CHANGELOG.md
|
|
6
|
+
Project-URL: Homepage, https://github.com/elliottower/reproducible-science
|
|
7
|
+
Project-URL: Issues, https://github.com/elliottower/reproducible-science/issues
|
|
8
|
+
Project-URL: Source, https://github.com/elliottower/reproducible-science/tree/main/packages/results
|
|
5
9
|
Author-email: Elliot Tower <elliot@elliottower.ai>
|
|
6
10
|
License: MIT
|
|
7
11
|
License-File: LICENSE
|
|
@@ -12,11 +16,19 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
12
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
17
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
18
|
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: provenance-core<0.4,>=0.3
|
|
15
20
|
Description-Content-Type: text/markdown
|
|
16
21
|
|
|
17
22
|
# results
|
|
18
23
|
|
|
19
|
-
|
|
24
|
+
[](https://pypi.org/project/results-cli/)
|
|
25
|
+
[](https://pypi.org/project/results-cli/)
|
|
26
|
+
[](https://github.com/elliottower/reproducible-science/blob/main/LICENSE)
|
|
27
|
+
[](https://elliottower.github.io/reproducible-science/)
|
|
28
|
+
|
|
29
|
+
Seal inputs, record outputs, and bind a paper's claims to the runs behind them.
|
|
30
|
+
|
|
31
|
+
Part of [reproducible-science](https://github.com/elliottower/reproducible-science) alongside `repro`, `citations` and `prereg` — see the [documentation](https://elliottower.github.io/reproducible-science/tools/results/).
|
|
20
32
|
|
|
21
33
|
## Install
|
|
22
34
|
|
|
@@ -109,17 +121,35 @@ whether it should have.
|
|
|
109
121
|
|
|
110
122
|
## Claude Code
|
|
111
123
|
|
|
112
|
-
`plugin/` is a Claude Code plugin
|
|
124
|
+
`plugin/` is a Claude Code plugin. Three surfaces, because each catches a different failure:
|
|
125
|
+
the hook catches what the model does not think to do, the skill catches what you did not know
|
|
126
|
+
to ask for, and the command is there for when you want the answer now.
|
|
127
|
+
|
|
128
|
+
| surface | fires |
|
|
129
|
+
|---|---|
|
|
130
|
+
| hook | when a number enters a manuscript that no recorded claim names |
|
|
131
|
+
| skill | when Claude judges the situation calls for sealing inputs, recording outputs, binding a paper's claims to runs |
|
|
132
|
+
| command | when you type `/results-check` |
|
|
133
|
+
|
|
134
|
+
**Why the hook.** The address is on screen while the sentence is being written and gone immediately afterward. A number recovered later has to be matched by its digits, and a value reported to two or three significant figures matches something in an artifact of any size.
|
|
135
|
+
|
|
136
|
+
It reports and never blocks, and stays silent in a project with no `.results/` ledger.
|
|
113
137
|
|
|
114
138
|
```bash
|
|
115
|
-
/plugin marketplace add elliottower/
|
|
116
|
-
/plugin install results@
|
|
139
|
+
/plugin marketplace add elliottower/reproducible-science
|
|
140
|
+
/plugin install results@reproducible-science
|
|
117
141
|
```
|
|
118
142
|
|
|
119
|
-
|
|
143
|
+
The plugin ships instructions and hooks, not binaries, so install the tool as well:
|
|
120
144
|
|
|
121
145
|
```bash
|
|
122
|
-
|
|
146
|
+
uv tool install results-cli # or: pip install results-cli
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
All four tools in one plugin, with every hook, skill and command:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
/plugin install reproducible-science@reproducible-science
|
|
123
153
|
```
|
|
124
154
|
|
|
125
155
|
MIT licensed.
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
# results
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://pypi.org/project/results-cli/)
|
|
4
|
+
[](https://pypi.org/project/results-cli/)
|
|
5
|
+
[](https://github.com/elliottower/reproducible-science/blob/main/LICENSE)
|
|
6
|
+
[](https://elliottower.github.io/reproducible-science/)
|
|
7
|
+
|
|
8
|
+
Seal inputs, record outputs, and bind a paper's claims to the runs behind them.
|
|
9
|
+
|
|
10
|
+
Part of [reproducible-science](https://github.com/elliottower/reproducible-science) alongside `repro`, `citations` and `prereg` — see the [documentation](https://elliottower.github.io/reproducible-science/tools/results/).
|
|
4
11
|
|
|
5
12
|
## Install
|
|
6
13
|
|
|
@@ -93,17 +100,35 @@ whether it should have.
|
|
|
93
100
|
|
|
94
101
|
## Claude Code
|
|
95
102
|
|
|
96
|
-
`plugin/` is a Claude Code plugin
|
|
103
|
+
`plugin/` is a Claude Code plugin. Three surfaces, because each catches a different failure:
|
|
104
|
+
the hook catches what the model does not think to do, the skill catches what you did not know
|
|
105
|
+
to ask for, and the command is there for when you want the answer now.
|
|
106
|
+
|
|
107
|
+
| surface | fires |
|
|
108
|
+
|---|---|
|
|
109
|
+
| hook | when a number enters a manuscript that no recorded claim names |
|
|
110
|
+
| skill | when Claude judges the situation calls for sealing inputs, recording outputs, binding a paper's claims to runs |
|
|
111
|
+
| command | when you type `/results-check` |
|
|
112
|
+
|
|
113
|
+
**Why the hook.** The address is on screen while the sentence is being written and gone immediately afterward. A number recovered later has to be matched by its digits, and a value reported to two or three significant figures matches something in an artifact of any size.
|
|
114
|
+
|
|
115
|
+
It reports and never blocks, and stays silent in a project with no `.results/` ledger.
|
|
97
116
|
|
|
98
117
|
```bash
|
|
99
|
-
/plugin marketplace add elliottower/
|
|
100
|
-
/plugin install results@
|
|
118
|
+
/plugin marketplace add elliottower/reproducible-science
|
|
119
|
+
/plugin install results@reproducible-science
|
|
101
120
|
```
|
|
102
121
|
|
|
103
|
-
|
|
122
|
+
The plugin ships instructions and hooks, not binaries, so install the tool as well:
|
|
104
123
|
|
|
105
124
|
```bash
|
|
106
|
-
|
|
125
|
+
uv tool install results-cli # or: pip install results-cli
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
All four tools in one plugin, with every hook, skill and command:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
/plugin install reproducible-science@reproducible-science
|
|
107
132
|
```
|
|
108
133
|
|
|
109
134
|
MIT licensed.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "results",
|
|
3
|
+
"description": "Seal a run's inputs, record what it produced, bind every number in a manuscript to the run behind it as it is written, and verify the chain",
|
|
4
|
+
"version": "0.2.0",
|
|
5
|
+
"author": {
|
|
6
|
+
"name": "Elliot Tower",
|
|
7
|
+
"email": "elliot@elliottower.ai"
|
|
8
|
+
},
|
|
9
|
+
"repository": "https://github.com/elliottower/reproducible-science/tree/main/packages/results",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"keywords": [
|
|
12
|
+
"claims",
|
|
13
|
+
"manuscript",
|
|
14
|
+
"open science",
|
|
15
|
+
"provenance",
|
|
16
|
+
"reproducibility",
|
|
17
|
+
"results"
|
|
18
|
+
]
|
|
19
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# The `results` Claude Code plugin
|
|
2
|
+
|
|
3
|
+
Does every number in the manuscript name the run that produced it?
|
|
4
|
+
|
|
5
|
+
Three surfaces, because each catches a different failure. The hook catches what the model
|
|
6
|
+
does not think to do, the skill catches what the author did not know to ask for, and the
|
|
7
|
+
command is there for when you want the answer now.
|
|
8
|
+
|
|
9
|
+
| | |
|
|
10
|
+
|---|---|
|
|
11
|
+
| **hook** `hooks/unbound_numbers.py` | fires when a number enters a manuscript that no recorded claim names |
|
|
12
|
+
| **skill** `skills/results/SKILL.md` | fires when Claude judges the tool relevant |
|
|
13
|
+
| **command** `/results-check` | fires when you type it |
|
|
14
|
+
|
|
15
|
+
## Why the hook
|
|
16
|
+
|
|
17
|
+
Recovering a binding after the fact does not work. Measured against 45 CORE-Bench capsules, values precise enough to identify are found about three times in four, and values reported to two or three significant figures cannot be identified at all, because a short number matches something in an artifact of any size. While writing, the address is on screen.
|
|
18
|
+
|
|
19
|
+
It reports and never blocks, exits zero on every failure path, and stays silent in a project
|
|
20
|
+
with no `.results/` ledger — nothing to check means nothing said.
|
|
21
|
+
|
|
22
|
+
## What the skill changes
|
|
23
|
+
|
|
24
|
+
Claude seals a run's inputs before computing, records the outputs after, and binds a number to its run as the sentence is written rather than leaving it to be traced later.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
/plugin marketplace add elliottower/reproducible-science
|
|
30
|
+
/plugin install results@reproducible-science
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The plugin ships instructions and hooks, not binaries. Install the tool as well:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uv tool install results-cli
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Against a checkout:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
/plugin marketplace add ~/Documents/GitHub/reproducible-science
|
|
43
|
+
/plugin install results@reproducible-science
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## The other three
|
|
47
|
+
|
|
48
|
+
`prereg`, `citations`, `results` and `repro` are one lifecycle, and each plugin ships
|
|
49
|
+
separately so you can take only what you use. `reproducible-science@reproducible-science`
|
|
50
|
+
installs all of them at once.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Does the results chain verify, and is every number in the manuscript bound to a run?
|
|
3
|
+
argument-hint: [manuscript]
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Two checks, reported together.
|
|
7
|
+
|
|
8
|
+
**The chain.** Run `results verify --files`. Report any event whose hash no longer matches,
|
|
9
|
+
any claim naming a run that is not in the ledger, and any output file that has changed since
|
|
10
|
+
it was recorded. A broken chain is reported before anything else, since it invalidates what
|
|
11
|
+
follows.
|
|
12
|
+
|
|
13
|
+
**The manuscript.** Run `results coverage $1`, defaulting to the newest manuscript source in
|
|
14
|
+
this repository and saying which you chose. Report:
|
|
15
|
+
|
|
16
|
+
- how many of the author's own numbers are bound to a run, as a count and a share;
|
|
17
|
+
- the unbound ones that state a result, grouped by line, sample sizes and denominators
|
|
18
|
+
first — an error there survives review;
|
|
19
|
+
- how many sit beside a citation and so belong to the work cited;
|
|
20
|
+
- how many owe no claim at all, named once as a group.
|
|
21
|
+
|
|
22
|
+
For each unbound result, say what would bind it, naming the run if the ledger already holds
|
|
23
|
+
one that produced it. Record nothing unless asked.
|
|
24
|
+
|
|
25
|
+
If there is no ledger, say that `results init` starts one.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Notice when a number enters a manuscript without a run behind it.
|
|
3
|
+
|
|
4
|
+
Runs as a Claude Code PostToolUse hook. Reads the tool call on stdin, works out which numbers
|
|
5
|
+
the edit added to a manuscript, and reports any that no recorded claim names.
|
|
6
|
+
|
|
7
|
+
The reason this is a hook rather than an instruction. Binding a claim to a run is a separate
|
|
8
|
+
deliberate act, and separate deliberate acts do not happen -- literate programming solved this
|
|
9
|
+
problem completely in 1984 and lost anyway. The address is on screen while the sentence is
|
|
10
|
+
written and gone immediately after, so the moment to record it is that moment, and something
|
|
11
|
+
has to notice when the moment passes.
|
|
12
|
+
|
|
13
|
+
Design constraints, in order:
|
|
14
|
+
|
|
15
|
+
1. Never break the session. Every failure exits 0 in silence. A hook that can halt the work
|
|
16
|
+
it observes gets turned off, and a hook that is off notices nothing.
|
|
17
|
+
2. Never block. This witnesses and reports; it does not refuse an edit.
|
|
18
|
+
3. Say nothing when there is nothing to say. A hook that fires on every edit is noise, and
|
|
19
|
+
noise gets muted. It speaks only when a manuscript gained a number nothing backs.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import pathlib
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
# One source of truth for what counts as a number and what owes no run. The hook runs as
|
|
31
|
+
# a bare script, so the package may not be importable; the fallbacks below keep it
|
|
32
|
+
# working rather than silently doing nothing.
|
|
33
|
+
from results.manuscript import LAYOUT, NUMBER, SUFFIXES, constraining_digits, needs_no_claim
|
|
34
|
+
except ImportError: # pragma: no cover - exercised only outside an installed environment
|
|
35
|
+
NUMBER = re.compile(
|
|
36
|
+
r"(?<![\w.])(?:(?<![-‐‑‒–—])[-+])?"
|
|
37
|
+
r"\d(?:,?\d)*(?:\.\d+)?(?:[eE][-+]?\d+)?(?![\w])"
|
|
38
|
+
)
|
|
39
|
+
LAYOUT = re.compile(
|
|
40
|
+
r"\\(?:vspace|hspace|setlength|includegraphics|multirow|multicolumn|cmidrule|cline"
|
|
41
|
+
r"|rule|label|ref|eqref|cite[a-z]*|bibitem|newcommand|usepackage|documentclass)"
|
|
42
|
+
r"\s*(?:\[[^\]]*\])?\s*(?:\{[^{}]*\})*",
|
|
43
|
+
re.I,
|
|
44
|
+
)
|
|
45
|
+
SUFFIXES = {".tex", ".md", ".rmd", ".qmd", ".typ", ".org", ".rst", ".txt"}
|
|
46
|
+
_CONSTANTS = {
|
|
47
|
+
"0",
|
|
48
|
+
"1",
|
|
49
|
+
"2",
|
|
50
|
+
"3",
|
|
51
|
+
"4",
|
|
52
|
+
"5",
|
|
53
|
+
"10",
|
|
54
|
+
"100",
|
|
55
|
+
"1000",
|
|
56
|
+
"1.96",
|
|
57
|
+
"0.05",
|
|
58
|
+
"0.01",
|
|
59
|
+
"0.001",
|
|
60
|
+
"95",
|
|
61
|
+
"99",
|
|
62
|
+
"0.5",
|
|
63
|
+
"0.95",
|
|
64
|
+
"2.5",
|
|
65
|
+
"97.5",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
def constraining_digits(printed: str) -> int:
|
|
69
|
+
body = printed.lstrip("-+").replace(",", "")
|
|
70
|
+
if "." in body:
|
|
71
|
+
return max(1, len(body.replace(".", "").lstrip("0")))
|
|
72
|
+
return max(1, len(body.strip("0") or "0"))
|
|
73
|
+
|
|
74
|
+
def needs_no_claim(printed: str, line: str, at: int) -> str | None:
|
|
75
|
+
if printed in _CONSTANTS:
|
|
76
|
+
return "a constant of the formula"
|
|
77
|
+
if constraining_digits(printed) < 2:
|
|
78
|
+
return "a single digit"
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
#: Never read a ledger larger than this. A hook runs on every edit and must stay cheap.
|
|
83
|
+
MAX_LEDGER_BYTES = 8 * 1024 * 1024
|
|
84
|
+
|
|
85
|
+
#: How many unbound values to name before summarising the rest.
|
|
86
|
+
MAX_SHOWN = 8
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def added_numbers(payload: dict) -> set[str]:
|
|
90
|
+
"""Numbers this edit put into the file that were not in the text it replaced."""
|
|
91
|
+
tool_input = payload.get("tool_input") or {}
|
|
92
|
+
before, after = "", ""
|
|
93
|
+
if "content" in tool_input:
|
|
94
|
+
after = str(tool_input["content"])
|
|
95
|
+
elif "edits" in tool_input:
|
|
96
|
+
for edit in tool_input["edits"] or []:
|
|
97
|
+
before += str(edit.get("old_string", ""))
|
|
98
|
+
after += str(edit.get("new_string", ""))
|
|
99
|
+
else:
|
|
100
|
+
before = str(tool_input.get("old_string", ""))
|
|
101
|
+
after = str(tool_input.get("new_string", ""))
|
|
102
|
+
|
|
103
|
+
def owing(text: str) -> set[str]:
|
|
104
|
+
stripped = LAYOUT.sub(" ", text)
|
|
105
|
+
found = set()
|
|
106
|
+
for line in stripped.splitlines():
|
|
107
|
+
for match in NUMBER.finditer(line):
|
|
108
|
+
if needs_no_claim(match.group(0), line, match.start()) is None:
|
|
109
|
+
found.add(match.group(0))
|
|
110
|
+
return found
|
|
111
|
+
|
|
112
|
+
return owing(after) - owing(before)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def bound_values(start: pathlib.Path) -> tuple[set[str], pathlib.Path] | None:
|
|
116
|
+
"""Numbers named by a recorded claim, and the ledger naming them, or None if untracked."""
|
|
117
|
+
for directory in [start, *start.parents]:
|
|
118
|
+
ledger = directory / ".results" / "ledger.jsonl"
|
|
119
|
+
if not ledger.is_file():
|
|
120
|
+
continue
|
|
121
|
+
try:
|
|
122
|
+
if ledger.stat().st_size > MAX_LEDGER_BYTES:
|
|
123
|
+
return None
|
|
124
|
+
found: set[str] = set()
|
|
125
|
+
for line in ledger.read_text(errors="replace").splitlines():
|
|
126
|
+
if '"claim"' not in line:
|
|
127
|
+
continue
|
|
128
|
+
record = json.loads(line)
|
|
129
|
+
if record.get("event") == "claim":
|
|
130
|
+
found.update(NUMBER.findall(record.get("claim", "")))
|
|
131
|
+
return found, ledger
|
|
132
|
+
except (OSError, ValueError):
|
|
133
|
+
return None
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def main() -> int:
|
|
138
|
+
try:
|
|
139
|
+
payload = json.load(sys.stdin)
|
|
140
|
+
except (ValueError, OSError):
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
path_text = (payload.get("tool_input") or {}).get("file_path", "")
|
|
144
|
+
if not path_text:
|
|
145
|
+
return 0
|
|
146
|
+
path = pathlib.Path(path_text)
|
|
147
|
+
if path.suffix.lower() not in SUFFIXES:
|
|
148
|
+
return 0
|
|
149
|
+
|
|
150
|
+
# Only ever walk up from the file itself. Falling back to the working directory finds the
|
|
151
|
+
# ledger of whichever repository the session started in, and reports an unrelated
|
|
152
|
+
# project's numbers as unbound.
|
|
153
|
+
tracked = bound_values(path.parent)
|
|
154
|
+
if tracked is None:
|
|
155
|
+
return 0
|
|
156
|
+
bound, ledger = tracked
|
|
157
|
+
|
|
158
|
+
unbound = sorted(added_numbers(payload) - bound)
|
|
159
|
+
if not unbound:
|
|
160
|
+
return 0
|
|
161
|
+
|
|
162
|
+
shown = ", ".join(unbound[:MAX_SHOWN])
|
|
163
|
+
if len(unbound) > MAX_SHOWN:
|
|
164
|
+
shown += f", and {len(unbound) - MAX_SHOWN} more"
|
|
165
|
+
message = (
|
|
166
|
+
f"{len(unbound)} number(s) entered {path.name} that no recorded claim names: {shown}. "
|
|
167
|
+
f"The source is in context now and will not be later. For each one that states a "
|
|
168
|
+
f"result, record it before moving on:\n"
|
|
169
|
+
f' results claim --run-id <run> --location "<where in the manuscript>" '
|
|
170
|
+
f'"<the sentence as it appears>"\n'
|
|
171
|
+
f"For a value quoted from another paper, or a parameter fixed by choice, say so once "
|
|
172
|
+
f"rather than each time. `results coverage {path.name}` audits the whole draft. "
|
|
173
|
+
f"Ledger: {ledger}"
|
|
174
|
+
)
|
|
175
|
+
json.dump(
|
|
176
|
+
{"hookSpecificOutput": {"hookEventName": "PostToolUse", "additionalContext": message}},
|
|
177
|
+
sys.stdout,
|
|
178
|
+
)
|
|
179
|
+
return 0
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
if __name__ == "__main__":
|
|
183
|
+
try:
|
|
184
|
+
sys.exit(main())
|
|
185
|
+
except Exception:
|
|
186
|
+
# Constraint 1. Nothing this hook can fail at is worth interrupting the work.
|
|
187
|
+
sys.exit(0)
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: results
|
|
3
|
+
description: Seal a run's inputs before computing, record outputs after, bind manuscript claims to specific runs, and verify the hash chain. Use before launching any computation whose output will appear in a paper, after a run completes, while writing a sentence that states a number, when a number in a manuscript needs tracing to its source, before submitting a draft, or when asked whether results are still what they were. Requires the `results` CLI (`uv tool install results-cli`).
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# results
|
|
7
|
+
|
|
8
|
+
Seal a run, record what it produced, and verify the chain.
|
|
9
|
+
|
|
10
|
+
## The rule that matters
|
|
11
|
+
|
|
12
|
+
**Seal before you run. Claim after you verify.**
|
|
13
|
+
|
|
14
|
+
A number in a manuscript is trustworthy when it names the run that produced it, the run names the
|
|
15
|
+
inputs that were sealed before it started, and the chain from input to claim has not been broken.
|
|
16
|
+
Every way this fails is a version of the result moving without a trace — a script edited after
|
|
17
|
+
the seal, an output overwritten between the run and the claim, a claim attached to a run that
|
|
18
|
+
no longer exists.
|
|
19
|
+
|
|
20
|
+
## Commands
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
results init # start tracking results here
|
|
24
|
+
results seal <file>... [--role input] # hash inputs before a run
|
|
25
|
+
results access <note> [--level ...] # record a data-access event
|
|
26
|
+
results run <file>... --run-id <id> # record outputs after a run
|
|
27
|
+
results claim <text> --run-id <id> # bind a manuscript claim to a run
|
|
28
|
+
results verify [--files] # check the chain and every hash it names
|
|
29
|
+
results coverage <manuscript> # which of a paper's numbers are bound
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## The workflow, in order
|
|
33
|
+
|
|
34
|
+
1. **`results init`** in the experiment directory.
|
|
35
|
+
2. **`results seal prereg.md script.py data.csv`** — hash every input before computing.
|
|
36
|
+
3. **`results access "downloaded zenodo metadata" --level "metadata only"`** — record what
|
|
37
|
+
was seen and when. This is the data-access taint the chain uses to distinguish confirmatory
|
|
38
|
+
from exploratory.
|
|
39
|
+
4. **Run the computation.**
|
|
40
|
+
5. **`results run output.json --run-id exp_001`** — hash the outputs.
|
|
41
|
+
6. **`results claim "ICC = 0.42" --run-id exp_001 --confirmatory --location "Table 2"`** —
|
|
42
|
+
bind the manuscript claim to the run.
|
|
43
|
+
7. **`results verify --files`** — check everything.
|
|
44
|
+
|
|
45
|
+
## Data-access levels
|
|
46
|
+
|
|
47
|
+
Four levels, in order of exposure:
|
|
48
|
+
|
|
49
|
+
| level | meaning |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `nothing seen` | no target data touched |
|
|
52
|
+
| `metadata only` | structure, region names, sample sizes — not outcomes |
|
|
53
|
+
| `structure seen` | data shape and distributions, but not the target variable |
|
|
54
|
+
| `outcomes seen` | the dependent variable was observed |
|
|
55
|
+
|
|
56
|
+
The level is recorded, not judged — log the honest one even when it is the damaging one.
|
|
57
|
+
|
|
58
|
+
## Exposure is not contamination
|
|
59
|
+
|
|
60
|
+
**What threatens a confirmatory reading is propagation, not exposure.** A context reading a
|
|
61
|
+
result is evidence that contamination was possible. Contamination is outcome information
|
|
62
|
+
reaching a consequential choice: selecting an analysis, changing an exclusion, editing the
|
|
63
|
+
code that computes the result, or abandoning an arm after a null.
|
|
64
|
+
|
|
65
|
+
| situation | disposition |
|
|
66
|
+
|---|---|
|
|
67
|
+
| plan frozen before execution, no premature exposure | confirmatory |
|
|
68
|
+
| results seen *after* every consequential choice was frozen | **confirmatory, exposure logged** |
|
|
69
|
+
| exposed context then changed the analysis or chose follow-up work | exploratory, for the affected decisions |
|
|
70
|
+
| a null caused an arm to be dropped | exploratory — continuation was data-contingent |
|
|
71
|
+
| a human saw results and then exercised discretion | exploratory unless those choices were already fixed |
|
|
72
|
+
|
|
73
|
+
Row two is the one that gets mishandled. A plan committed to git before the exposure cannot
|
|
74
|
+
be reached by it, so `results claim --confirmatory --frozen-at <commit>` records the claim
|
|
75
|
+
as confirmatory with the exposure logged. `verify` reports it separately from contested
|
|
76
|
+
claims. Without a freeze reference the tool refuses, because it cannot tell the two apart.
|
|
77
|
+
|
|
78
|
+
**Demote the affected decisions, never the study.** Blanket demotion is a category error, and
|
|
79
|
+
it is why exposures get hidden rather than logged: if disclosure costs everything, nobody
|
|
80
|
+
discloses. Scope the demotion to the choices the exposure could actually have reached.
|
|
81
|
+
|
|
82
|
+
**The boundary.** If outcome information reaches a human who then exercises relevant
|
|
83
|
+
discretion, non-propagation cannot be *demonstrated* — not because it occurred, but because
|
|
84
|
+
no infrastructure can show it did not. Those decisions do not get the protection available to
|
|
85
|
+
isolated computational contexts.
|
|
86
|
+
|
|
87
|
+
## Reading `results verify`
|
|
88
|
+
|
|
89
|
+
The chain check is pass/fail:
|
|
90
|
+
|
|
91
|
+
| | |
|
|
92
|
+
|---|---|
|
|
93
|
+
| `chain intact` | every event's prev_hash matches the hash of the line before it |
|
|
94
|
+
| `CHAIN BROKEN` | a line was edited, inserted, or deleted after it was written |
|
|
95
|
+
|
|
96
|
+
With `--files`, every sealed input and recorded output is re-hashed against its current state:
|
|
97
|
+
|
|
98
|
+
| | |
|
|
99
|
+
|---|---|
|
|
100
|
+
| `ok` | file matches its recorded hash |
|
|
101
|
+
| `CHANGED` | file was modified since it was sealed or recorded |
|
|
102
|
+
| `MISSING` | file no longer exists at the recorded path |
|
|
103
|
+
|
|
104
|
+
**`CHAIN BROKEN` means the ledger was tampered with.** It does not mean the results are wrong —
|
|
105
|
+
it means the evidence that they are right was damaged. Restore from git history.
|
|
106
|
+
|
|
107
|
+
## Claims
|
|
108
|
+
|
|
109
|
+
A claim names a run. A run names its outputs. The outputs were hashed when they were recorded.
|
|
110
|
+
This is the whole chain: manuscript → claim → run → output file → hash. `verify` walks it.
|
|
111
|
+
|
|
112
|
+
`--confirmatory` marks the claim as backed by a pre-registered hypothesis. Without it, the claim
|
|
113
|
+
is exploratory. The distinction is recorded, not enforced — the access timeline is what makes it
|
|
114
|
+
verifiable.
|
|
115
|
+
|
|
116
|
+
## Binding while writing
|
|
117
|
+
|
|
118
|
+
**When a number enters a manuscript, record what produced it before moving on.**
|
|
119
|
+
|
|
120
|
+
The address is on screen while the sentence is being written and gone immediately afterward.
|
|
121
|
+
Recovering it later is measurably hard: matching a printed value against an artifact by its
|
|
122
|
+
digits fails once the artifact is dense, and on one development repository a four-digit value
|
|
123
|
+
confirms at 94 per cent while a fabricated value of the same shape confirms at 93.8 per cent.
|
|
124
|
+
The values are genuinely there and the match shows nothing. What separates a match from a
|
|
125
|
+
coincidence is the address, and a finished paper carries none.
|
|
126
|
+
|
|
127
|
+
The plugin's hook watches edits to manuscript files in repositories that have run
|
|
128
|
+
`results init`. When an edit introduces a number no claim names, it says which. Answer it by
|
|
129
|
+
recording the claim, or by noting once that the value needs none.
|
|
130
|
+
|
|
131
|
+
A value needs no claim when it is a constant of a formula, a number quoted from another
|
|
132
|
+
paper, or a parameter fixed by choice rather than produced by computation. Say so once for
|
|
133
|
+
the document rather than each time it comes up.
|
|
134
|
+
|
|
135
|
+
## Auditing a draft
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
results coverage paper/paper_v23.tex # what is bound, and what is not
|
|
139
|
+
results coverage paper.tex --strict # non-zero exit if anything is unbound
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Run it before submitting. On a real manuscript with 21 recorded claims it reported 27 per
|
|
143
|
+
cent of the numbers bound, and among the unbound were sample-size denominators and interval
|
|
144
|
+
bounds — the class of number whose errors survive review.
|
|
145
|
+
|
|
146
|
+
## When to reach for this
|
|
147
|
+
|
|
148
|
+
- Before launching any computation whose output will be reported
|
|
149
|
+
- After a run completes, to record what it produced
|
|
150
|
+
- When a number in a manuscript needs tracing to its source run
|
|
151
|
+
- When asked whether results are still what they were
|
|
152
|
+
- In CI, as `results verify --files` and `results coverage <paper> --strict`
|
|
153
|
+
- Before submitting, to see which numbers nothing backs
|
|
154
|
+
|
|
155
|
+
## What it will not do
|
|
156
|
+
|
|
157
|
+
It cannot prove the computation was correct. A sealed script that produces the wrong answer
|
|
158
|
+
passes every check here. It also cannot prove the access timeline is honest — the events are
|
|
159
|
+
self-reported, and the tool records them without questioning. The chain is evidence, not proof.
|
|
160
|
+
|
|
161
|
+
## Where this sits
|
|
162
|
+
|
|
163
|
+
Four tools guard four moments, and each is weak without the others. A frozen plan over
|
|
164
|
+
unsealed inputs proves nothing; a sealed run whose numbers never reach the manuscript proves
|
|
165
|
+
nothing either.
|
|
166
|
+
|
|
167
|
+
| moment | tool | what it fixes |
|
|
168
|
+
|---|---|---|
|
|
169
|
+
| before you run | `prereg freeze` | the plan cannot be rewritten around the result |
|
|
170
|
+
| before you compute | `results seal` | the inputs are what you say they were |
|
|
171
|
+
| after a run | `results run` | the outputs are recorded and hashed |
|
|
172
|
+
| writing a number | `results claim` | the number names the run behind it |
|
|
173
|
+
| writing a quotation | a `claims/` entry | the passage is in the source |
|
|
174
|
+
| before submitting | `prereg check`, `results verify`, `citations verify` | nothing drifted |
|
|
175
|
+
|
|
176
|
+
Every tool named here is installed by `uv tool install reproducible-science`, so the commands
|
|
177
|
+
above are available whatever plugins are present.
|
|
178
|
+
|
|
179
|
+
Each tool also ships its own plugin, adding a hook that speaks when its moment passes
|
|
180
|
+
unrecorded and a `-check` command. Those commands exist only where the matching plugin is
|
|
181
|
+
installed; the CLI calls in the table always work.
|
|
@@ -4,7 +4,7 @@ requires = [ "hatchling" ]
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "results-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Seal a run, record what it produced, and verify the chain"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
keywords = [ "open science", "provenance", "reproducibility", "results" ]
|
|
@@ -18,7 +18,11 @@ classifiers = [
|
|
|
18
18
|
"Programming Language :: Python :: 3.13",
|
|
19
19
|
"Programming Language :: Python :: 3.14",
|
|
20
20
|
]
|
|
21
|
-
dependencies = [
|
|
21
|
+
dependencies = [ "provenance-core>=0.3,<0.4" ]
|
|
22
|
+
urls.Changelog = "https://github.com/elliottower/reproducible-science/blob/main/CHANGELOG.md"
|
|
23
|
+
urls.Homepage = "https://github.com/elliottower/reproducible-science"
|
|
24
|
+
urls.Issues = "https://github.com/elliottower/reproducible-science/issues"
|
|
25
|
+
urls.Source = "https://github.com/elliottower/reproducible-science/tree/main/packages/results"
|
|
22
26
|
scripts.results = "results.cli:main"
|
|
23
27
|
|
|
24
28
|
[dependency-groups]
|