results-cli 0.5.4__tar.gz → 0.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {results_cli-0.5.4 → results_cli-0.5.6}/PKG-INFO +4 -2
- {results_cli-0.5.4 → results_cli-0.5.6}/README.md +3 -1
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/.claude-plugin/plugin.json +1 -1
- {results_cli-0.5.4 → results_cli-0.5.6}/pyproject.toml +1 -1
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/audit.py +23 -2
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/cli.py +32 -3
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/record.py +57 -11
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_cli.py +82 -0
- results_cli-0.5.6/tests/test_verify_json.py +55 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/.gitignore +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/LICENSE +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/README.md +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/commands/results-check.md +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/hooks/hooks.json +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/hooks/unbound_numbers.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/plugin/skills/results/SKILL.md +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/__init__.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/ledger.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/manuscript.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/paths.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/stamps.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/src/results/timeline.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_annotate.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_cli_help.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_concurrency.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_coverage_and_hook.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_ledger.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_manuscript.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_review.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_tamper.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_timeline.py +0 -0
- {results_cli-0.5.4 → results_cli-0.5.6}/tests/test_timestamp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: results-cli
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.6
|
|
4
4
|
Summary: Seal a run, record what it produced, and verify the chain
|
|
5
5
|
Project-URL: Changelog, https://github.com/elliottower/reproducible-science/blob/main/CHANGELOG.md
|
|
6
6
|
Project-URL: Homepage, https://github.com/elliottower/reproducible-science
|
|
@@ -74,11 +74,13 @@ all checks passed.
|
|
|
74
74
|
| `results init` | Start tracking results here |
|
|
75
75
|
| `results seal <file>...` | Hash inputs before a run |
|
|
76
76
|
| `results access <note>` | Record a data-access event |
|
|
77
|
-
| `results run <file>...` | Record outputs after a run |
|
|
77
|
+
| `results run <file>...` | Record outputs after a run; `--holds` says the outputs hold metadata or structure and no outcome |
|
|
78
|
+
| `results label <run-id>` | Declare after the fact that a recorded run's outputs hold no outcome |
|
|
78
79
|
| `results claim <text>` | Bind a manuscript claim to a run |
|
|
79
80
|
| `results review <file>...` | Record that someone examined these files, and what they found |
|
|
80
81
|
| `results annotate <file>...` | Link these files to something outside the ledger; linked, not checked |
|
|
81
82
|
| `results verify` | Check the ledger chain and every hash it names |
|
|
83
|
+
| `results verify --json PATH` | Also write what it found to `PATH` as JSON: the chain's status, the number of events, and each recorded file with its state |
|
|
82
84
|
| `results timestamp` | Date the ledger's head outside the repository, and check earlier dates |
|
|
83
85
|
|
|
84
86
|
## The chain
|
|
@@ -53,11 +53,13 @@ all checks passed.
|
|
|
53
53
|
| `results init` | Start tracking results here |
|
|
54
54
|
| `results seal <file>...` | Hash inputs before a run |
|
|
55
55
|
| `results access <note>` | Record a data-access event |
|
|
56
|
-
| `results run <file>...` | Record outputs after a run |
|
|
56
|
+
| `results run <file>...` | Record outputs after a run; `--holds` says the outputs hold metadata or structure and no outcome |
|
|
57
|
+
| `results label <run-id>` | Declare after the fact that a recorded run's outputs hold no outcome |
|
|
57
58
|
| `results claim <text>` | Bind a manuscript claim to a run |
|
|
58
59
|
| `results review <file>...` | Record that someone examined these files, and what they found |
|
|
59
60
|
| `results annotate <file>...` | Link these files to something outside the ledger; linked, not checked |
|
|
60
61
|
| `results verify` | Check the ledger chain and every hash it names |
|
|
62
|
+
| `results verify --json PATH` | Also write what it found to `PATH` as JSON: the chain's status, the number of events, and each recorded file with its state |
|
|
61
63
|
| `results timestamp` | Date the ledger's head outside the repository, and check earlier dates |
|
|
62
64
|
|
|
63
65
|
## The chain
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "results",
|
|
3
3
|
"description": "Seal a run's inputs, record what it produced, bind every number in a manuscript to the run behind it as it is written, and verify the chain",
|
|
4
|
-
"version": "0.5.
|
|
4
|
+
"version": "0.5.6",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Elliot Tower",
|
|
7
7
|
"email": "elliot@elliottower.ai"
|
|
@@ -4,7 +4,7 @@ requires = [ "hatchling" ]
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "results-cli"
|
|
7
|
-
version = "0.5.
|
|
7
|
+
version = "0.5.6"
|
|
8
8
|
description = "Seal a run, record what it produced, and verify the chain"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
keywords = [ "open science", "provenance", "reproducibility", "results" ]
|
|
@@ -13,9 +13,10 @@ is why it is a separate command and not something verification does on its own.
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
+
import json
|
|
16
17
|
import pathlib
|
|
17
18
|
|
|
18
|
-
from provenance_core import anchor, sha256_of_tree, try_run
|
|
19
|
+
from provenance_core import anchor, atomic_write, sha256_of_tree, try_run
|
|
19
20
|
|
|
20
21
|
from results import ledger, manuscript, stamps
|
|
21
22
|
from results.paths import ledger_path, require_root
|
|
@@ -233,11 +234,23 @@ def timestamp() -> int:
|
|
|
233
234
|
return 0
|
|
234
235
|
|
|
235
236
|
|
|
236
|
-
def verify(check_files: bool) -> int:
|
|
237
|
+
def verify(check_files: bool, json_path: str | None = None) -> int:
|
|
238
|
+
"""Check the chain and, with `check_files`, every recorded file. With `json_path`, the
|
|
239
|
+
same findings are also written there as one JSON object: the chain's status and problems,
|
|
240
|
+
the number of events, and each recorded path with what was found at it."""
|
|
241
|
+
report: dict = {"files_checked": check_files, "files": []}
|
|
242
|
+
code = _verify(check_files, report)
|
|
243
|
+
if json_path:
|
|
244
|
+
atomic_write(json_path, json.dumps({**report, "exit": code}, indent=2) + "\n")
|
|
245
|
+
return code
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _verify(check_files: bool, report: dict) -> int:
|
|
237
249
|
root = require_root()
|
|
238
250
|
lp = ledger_path(root)
|
|
239
251
|
|
|
240
252
|
status, problems = ledger.verify(lp)
|
|
253
|
+
report |= {"chain": status.value, "problems": problems}
|
|
241
254
|
if status is not ledger.ChainStatus.INTACT:
|
|
242
255
|
# Name which failure it is. Truncated, edited and unattested have different causes and
|
|
243
256
|
# different remedies, and one banner for all three tells a reader nothing to act on.
|
|
@@ -262,6 +275,7 @@ def verify(check_files: bool) -> int:
|
|
|
262
275
|
return 1
|
|
263
276
|
|
|
264
277
|
events = ledger.read_ledger(lp)
|
|
278
|
+
report["events"] = len(events)
|
|
265
279
|
print(f"chain intact: {len(events)} events, anchored\n")
|
|
266
280
|
|
|
267
281
|
if state := in_history(lp):
|
|
@@ -315,10 +329,12 @@ def verify(check_files: bool) -> int:
|
|
|
315
329
|
if f["sha256"] not in recorded:
|
|
316
330
|
recorded.append(f["sha256"])
|
|
317
331
|
base = root.parent
|
|
332
|
+
found = report["files"]
|
|
318
333
|
for path, recorded in sorted(file_hashes.items()):
|
|
319
334
|
p = base / path
|
|
320
335
|
if not p.exists():
|
|
321
336
|
print(f" MISSING {path}")
|
|
337
|
+
found.append({"path": path, "state": "missing"})
|
|
322
338
|
drift += 1
|
|
323
339
|
continue
|
|
324
340
|
if kinds.get(path) == "tree":
|
|
@@ -328,12 +344,14 @@ def verify(check_files: bool) -> int:
|
|
|
328
344
|
# that never happened.
|
|
329
345
|
print(f" NOT A TREE {path}")
|
|
330
346
|
print(" sealed as a directory; what is there now is not one")
|
|
347
|
+
found.append({"path": path, "state": "not a tree"})
|
|
331
348
|
drift += 1
|
|
332
349
|
continue
|
|
333
350
|
actual = sha256_of_tree(p)[0]
|
|
334
351
|
elif p.is_dir():
|
|
335
352
|
print(f" NOT A FILE {path}")
|
|
336
353
|
print(" sealed as a file; what is there now is a directory")
|
|
354
|
+
found.append({"path": path, "state": "not a file"})
|
|
337
355
|
drift += 1
|
|
338
356
|
continue
|
|
339
357
|
else:
|
|
@@ -344,13 +362,16 @@ def verify(check_files: bool) -> int:
|
|
|
344
362
|
# so the path was sealed, changed, and sealed again.
|
|
345
363
|
print(f" RESEALED {path}")
|
|
346
364
|
print(f" recorded {len(recorded)} different hashes for this path")
|
|
365
|
+
found.append({"path": path, "state": "resealed"})
|
|
347
366
|
drift += 1
|
|
348
367
|
else:
|
|
349
368
|
print(f" ok {path}")
|
|
369
|
+
found.append({"path": path, "state": "ok"})
|
|
350
370
|
else:
|
|
351
371
|
print(f" CHANGED {path}")
|
|
352
372
|
print(f" sealed {recorded[-1][:16]}…")
|
|
353
373
|
print(f" now {actual[:16]}…")
|
|
374
|
+
found.append({"path": path, "state": "changed"})
|
|
354
375
|
drift += 1
|
|
355
376
|
if drift:
|
|
356
377
|
print(f"\n{drift} file(s) changed or missing since they were recorded.")
|
|
@@ -4,6 +4,7 @@ results init start tracking results here
|
|
|
4
4
|
results seal <file>... hash inputs before a run (prereg, script, data)
|
|
5
5
|
results access <note> record a data-access event (what you looked at, when)
|
|
6
6
|
results run <file>... record outputs after a run completes
|
|
7
|
+
results label <run-id> declare after the fact that a recorded run's outputs hold no outcome
|
|
7
8
|
results claim <text> bind a manuscript claim to a run's output
|
|
8
9
|
results review <file>... record that someone examined these files, and what they found
|
|
9
10
|
results annotate <file>... link these files to something outside the ledger; linked, not checked
|
|
@@ -33,7 +34,7 @@ from results.paths import find_root as find_root
|
|
|
33
34
|
from results.paths import ledger_path as ledger_path
|
|
34
35
|
from results.paths import record_path as record_path
|
|
35
36
|
from results.paths import require_root as require_root
|
|
36
|
-
from results.record import ACCESS_LEVELS, VERDICTS
|
|
37
|
+
from results.record import ACCESS_LEVELS, RUN_HOLDS, VERDICTS
|
|
37
38
|
from results.timeline import first_outcomes_seen as first_outcomes_seen
|
|
38
39
|
from results.timeline import first_run_timestamp as first_run_timestamp
|
|
39
40
|
from results.timeline import freeze_timestamp as freeze_timestamp
|
|
@@ -53,7 +54,13 @@ def cmd_access(a) -> int:
|
|
|
53
54
|
|
|
54
55
|
|
|
55
56
|
def cmd_run(a) -> int:
|
|
56
|
-
return record.run(
|
|
57
|
+
return record.run(
|
|
58
|
+
a.files, a.run_id, a.note, getattr(a, "anyway", False), getattr(a, "holds", None)
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def cmd_label(a) -> int:
|
|
63
|
+
return record.label(a.run_id, a.holds, a.note)
|
|
57
64
|
|
|
58
65
|
|
|
59
66
|
def cmd_claim(a) -> int:
|
|
@@ -84,7 +91,7 @@ def cmd_reanchor(a) -> int:
|
|
|
84
91
|
|
|
85
92
|
|
|
86
93
|
def cmd_verify(a) -> int:
|
|
87
|
-
return audit.verify(a.files)
|
|
94
|
+
return audit.verify(a.files, a.json)
|
|
88
95
|
|
|
89
96
|
|
|
90
97
|
def cmd_timestamp(a) -> int:
|
|
@@ -128,6 +135,12 @@ def _main(argv: list[str] | None = None) -> int:
|
|
|
128
135
|
r.add_argument("files", nargs="+")
|
|
129
136
|
r.add_argument("--run-id", required=True, help="a name for this run")
|
|
130
137
|
r.add_argument("--note", help="what this run computed")
|
|
138
|
+
r.add_argument(
|
|
139
|
+
"--holds",
|
|
140
|
+
help=f"what the outputs hold, where they hold no outcome: one of {', '.join(RUN_HOLDS)}. "
|
|
141
|
+
"For a panel or feasibility query whose counts are reported. `prereg freeze` does not "
|
|
142
|
+
"count such a run as a result; a run recorded without this is taken to hold results",
|
|
143
|
+
)
|
|
131
144
|
r.add_argument(
|
|
132
145
|
"--anyway",
|
|
133
146
|
action="store_true",
|
|
@@ -136,6 +149,17 @@ def _main(argv: list[str] | None = None) -> int:
|
|
|
136
149
|
)
|
|
137
150
|
r.set_defaults(fn=cmd_run)
|
|
138
151
|
|
|
152
|
+
lb = sub.add_parser(
|
|
153
|
+
"label",
|
|
154
|
+
help="declare after the fact that a recorded run's outputs hold no outcome",
|
|
155
|
+
)
|
|
156
|
+
lb.add_argument("run_id", help="the run, recorded without --holds")
|
|
157
|
+
lb.add_argument("--holds", required=True, help=f"one of: {', '.join(RUN_HOLDS)}")
|
|
158
|
+
lb.add_argument(
|
|
159
|
+
"--note", required=True, help="why the run was not labelled when it was recorded"
|
|
160
|
+
)
|
|
161
|
+
lb.set_defaults(fn=cmd_label)
|
|
162
|
+
|
|
139
163
|
cl = sub.add_parser("claim", help="bind a manuscript claim to a run")
|
|
140
164
|
cl.add_argument("text", help="the claim, as it appears in the manuscript")
|
|
141
165
|
cl.add_argument("--run-id", required=True, help="which run backs this claim")
|
|
@@ -202,6 +226,11 @@ def _main(argv: list[str] | None = None) -> int:
|
|
|
202
226
|
action="store_true",
|
|
203
227
|
help="also check that sealed/output files still match their hashes",
|
|
204
228
|
)
|
|
229
|
+
v.add_argument(
|
|
230
|
+
"--json",
|
|
231
|
+
metavar="PATH",
|
|
232
|
+
help="also write the findings to this file as JSON: the chain, and each recorded file",
|
|
233
|
+
)
|
|
205
234
|
v.set_defaults(fn=cmd_verify)
|
|
206
235
|
|
|
207
236
|
cv = sub.add_parser("coverage", help="how many of a manuscript's numbers are bound to a run")
|
|
@@ -155,7 +155,18 @@ def access(note: str, level: str) -> int:
|
|
|
155
155
|
return 0
|
|
156
156
|
|
|
157
157
|
|
|
158
|
-
|
|
158
|
+
#: What a run's outputs can be declared to hold where they hold no outcome. A run recorded
|
|
159
|
+
#: without one is taken to hold results, which is what a run meant before the field existed.
|
|
160
|
+
RUN_HOLDS = ["metadata only", "structure seen"]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def run(
|
|
164
|
+
files: list[str], run_id: str, note: str | None, anyway: bool, holds: str | None = None
|
|
165
|
+
) -> int:
|
|
166
|
+
if holds is not None and holds not in RUN_HOLDS:
|
|
167
|
+
print(f"--holds must be one of: {', '.join(RUN_HOLDS)}")
|
|
168
|
+
print("a run whose outputs hold outcomes is recorded without --holds.")
|
|
169
|
+
return 1
|
|
159
170
|
root = require_root()
|
|
160
171
|
lp = ledger_path(root)
|
|
161
172
|
existing = ledger.read_ledger(lp)
|
|
@@ -176,21 +187,56 @@ def run(files: list[str], run_id: str, note: str | None, anyway: bool) -> int:
|
|
|
176
187
|
return 1
|
|
177
188
|
digest = ledger.sha256_of_file(p)
|
|
178
189
|
outputs.append({"path": record_path(p, root), "sha256": digest})
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
190
|
+
event = {
|
|
191
|
+
"event": "run",
|
|
192
|
+
"run_id": run_id,
|
|
193
|
+
"outputs": outputs,
|
|
194
|
+
"note": note or "",
|
|
195
|
+
}
|
|
196
|
+
if holds is not None:
|
|
197
|
+
# Written with the run and never afterwards: no command labels a run already recorded,
|
|
198
|
+
# so the declaration is in the chain before anything that reads it.
|
|
199
|
+
event["holds"] = holds
|
|
200
|
+
ledger.append_event(lp, event)
|
|
201
|
+
print(f"run {run_id}: {len(outputs)} output(s){f', holding {holds}' if holds else ''}")
|
|
189
202
|
for o in outputs:
|
|
190
203
|
print(f" {o['sha256'][:16]}… {o['path']}")
|
|
191
204
|
return 0
|
|
192
205
|
|
|
193
206
|
|
|
207
|
+
def label(run_id: str, holds: str, note: str) -> int:
|
|
208
|
+
"""Declare, after the fact, that a recorded run's outputs hold no outcome.
|
|
209
|
+
|
|
210
|
+
For the run recorded without `--holds` by oversight. The declaration is an event of its
|
|
211
|
+
own, so the ledger shows it was made after the run and what else had been recorded by
|
|
212
|
+
then; the run event is not rewritten. A run is declared once and a label is never changed.
|
|
213
|
+
"""
|
|
214
|
+
if holds not in RUN_HOLDS:
|
|
215
|
+
print(f"--holds must be one of: {', '.join(RUN_HOLDS)}")
|
|
216
|
+
return 1
|
|
217
|
+
if not note.strip():
|
|
218
|
+
print("a note is required: say why the run was not labelled when it was recorded.")
|
|
219
|
+
return 1
|
|
220
|
+
root = require_root()
|
|
221
|
+
lp = ledger_path(root)
|
|
222
|
+
events = ledger.read_ledger(lp)
|
|
223
|
+
runs = [e for e in events if e.get("event") == "run" and e.get("run_id") == run_id]
|
|
224
|
+
if not runs:
|
|
225
|
+
print(f"no run '{run_id}' in the ledger.")
|
|
226
|
+
return 1
|
|
227
|
+
if len(runs) > 1:
|
|
228
|
+
print(f"run id '{run_id}' names {len(runs)} runs, and a label has to name one.")
|
|
229
|
+
return 1
|
|
230
|
+
declared = [e for e in events if e.get("event") == "label" and e.get("run_id") == run_id]
|
|
231
|
+
if "holds" in runs[0] or declared:
|
|
232
|
+
print(f"run '{run_id}' already says what it holds; a label is not changed.")
|
|
233
|
+
return 1
|
|
234
|
+
ledger.append_event(lp, {"event": "label", "run_id": run_id, "holds": holds, "note": note})
|
|
235
|
+
print(f"run {run_id}: declared after the fact as holding {holds}")
|
|
236
|
+
print("`prereg freeze` reports a declaration made after the run was recorded as such.")
|
|
237
|
+
return 0
|
|
238
|
+
|
|
239
|
+
|
|
194
240
|
def claim(
|
|
195
241
|
text: str,
|
|
196
242
|
run_id: str,
|
|
@@ -96,6 +96,88 @@ def test_run_records_outputs(tmp_path):
|
|
|
96
96
|
assert runs[0]["run_id"] == "run_001"
|
|
97
97
|
|
|
98
98
|
|
|
99
|
+
def test_run_records_what_its_outputs_hold_only_when_told(tmp_path):
|
|
100
|
+
run_cli("init", cwd=tmp_path)
|
|
101
|
+
(tmp_path / "counts.json").write_text('{"pairs": 16}\n')
|
|
102
|
+
(tmp_path / "results.json").write_text('{"icc": 0.42}\n')
|
|
103
|
+
labelled = run_cli(
|
|
104
|
+
"run", "counts.json", "--run-id", "panel", "--holds", "metadata only", cwd=tmp_path
|
|
105
|
+
)
|
|
106
|
+
assert labelled.returncode == 0
|
|
107
|
+
assert "holding metadata only" in labelled.stdout
|
|
108
|
+
assert run_cli("run", "results.json", "--run-id", "exp", cwd=tmp_path).returncode == 0
|
|
109
|
+
runs = {
|
|
110
|
+
e["run_id"]: e
|
|
111
|
+
for e in ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
112
|
+
if e["event"] == "run"
|
|
113
|
+
}
|
|
114
|
+
assert runs["panel"]["holds"] == "metadata only"
|
|
115
|
+
assert "holds" not in runs["exp"]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_run_refuses_a_label_that_does_not_mean_no_outcome(tmp_path):
|
|
119
|
+
run_cli("init", cwd=tmp_path)
|
|
120
|
+
(tmp_path / "out.json").write_text("{}\n")
|
|
121
|
+
for label in ("outcomes seen", "nothing seen", "metadata"):
|
|
122
|
+
r = run_cli("run", "out.json", "--run-id", "x", "--holds", label, cwd=tmp_path)
|
|
123
|
+
assert r.returncode == 1
|
|
124
|
+
assert "--holds must be one of: metadata only, structure seen" in r.stdout
|
|
125
|
+
events = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
126
|
+
assert [e["event"] for e in events] == ["init"]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_label_declares_a_recorded_run_after_the_fact_as_its_own_event(tmp_path):
|
|
130
|
+
run_cli("init", cwd=tmp_path)
|
|
131
|
+
(tmp_path / "counts.json").write_text('{"pairs": 16}\n')
|
|
132
|
+
run_cli("run", "counts.json", "--run-id", "panel", cwd=tmp_path)
|
|
133
|
+
before = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
134
|
+
r = run_cli(
|
|
135
|
+
"label", "panel", "--holds", "metadata only", "--note", "flag forgotten", cwd=tmp_path
|
|
136
|
+
)
|
|
137
|
+
assert r.returncode == 0
|
|
138
|
+
assert "declared after the fact as holding metadata only" in r.stdout
|
|
139
|
+
after = ledger.read_ledger(tmp_path / ".results" / "ledger.jsonl")
|
|
140
|
+
assert after[: len(before)] == before, "the run event is not rewritten"
|
|
141
|
+
assert after[-1]["event"] == "label"
|
|
142
|
+
assert (after[-1]["run_id"], after[-1]["holds"]) == ("panel", "metadata only")
|
|
143
|
+
assert run_cli("verify", cwd=tmp_path).returncode == 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_label_refuses_what_it_cannot_declare_and_records_nothing(tmp_path):
|
|
147
|
+
run_cli("init", cwd=tmp_path)
|
|
148
|
+
(tmp_path / "a.json").write_text("{}\n")
|
|
149
|
+
run_cli("run", "a.json", "--run-id", "held", "--holds", "structure seen", cwd=tmp_path)
|
|
150
|
+
run_cli("run", "a.json", "--run-id", "twice", cwd=tmp_path)
|
|
151
|
+
run_cli("run", "a.json", "--run-id", "twice", "--anyway", cwd=tmp_path)
|
|
152
|
+
run_cli("run", "a.json", "--run-id", "plain", cwd=tmp_path)
|
|
153
|
+
path = tmp_path / ".results" / "ledger.jsonl"
|
|
154
|
+
n = len(ledger.read_ledger(path))
|
|
155
|
+
|
|
156
|
+
def refused(*args):
|
|
157
|
+
r = run_cli("label", *args, cwd=tmp_path)
|
|
158
|
+
assert r.returncode == 1
|
|
159
|
+
return r.stdout
|
|
160
|
+
|
|
161
|
+
assert "no run 'missing'" in refused("missing", "--holds", "metadata only", "--note", "x")
|
|
162
|
+
assert "already says what it holds" in refused(
|
|
163
|
+
"held", "--holds", "metadata only", "--note", "x"
|
|
164
|
+
)
|
|
165
|
+
assert "names 2 runs" in refused("twice", "--holds", "metadata only", "--note", "x")
|
|
166
|
+
assert "--holds must be one of" in refused("plain", "--holds", "outcomes seen", "--note", "x")
|
|
167
|
+
assert "a note is required" in refused("plain", "--holds", "metadata only", "--note", " ")
|
|
168
|
+
assert len(ledger.read_ledger(path)) == n
|
|
169
|
+
|
|
170
|
+
assert (
|
|
171
|
+
run_cli(
|
|
172
|
+
"label", "plain", "--holds", "metadata only", "--note", "x", cwd=tmp_path
|
|
173
|
+
).returncode
|
|
174
|
+
== 0
|
|
175
|
+
)
|
|
176
|
+
assert "already says what it holds" in refused(
|
|
177
|
+
"plain", "--holds", "structure seen", "--note", "x"
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
99
181
|
def test_run_refuses_a_duplicate_id(tmp_path):
|
|
100
182
|
"""A warning was not enough. Both the claim-time refusal and `verify`'s contested list
|
|
101
183
|
resolve an id to a single timestamp, so two runs sharing an id let a confirmatory claim
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""`results verify --json PATH` writes what the run found beside what it prints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from results.cli import main
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def recorded(tmp_path, monkeypatch):
|
|
13
|
+
monkeypatch.chdir(tmp_path)
|
|
14
|
+
(tmp_path / "inputs.json").write_text("[1, 2, 3]")
|
|
15
|
+
(tmp_path / "results.json").write_text('{"total": 6}')
|
|
16
|
+
assert main(["init"]) == 0
|
|
17
|
+
assert main(["seal", "inputs.json"]) == 0
|
|
18
|
+
assert main(["run", "results.json", "--run-id", "r1"]) == 0
|
|
19
|
+
return tmp_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def found(root, *flags):
|
|
23
|
+
code = main(["verify", *flags, "--json", str(root / "found.json")])
|
|
24
|
+
return code, json.loads((root / "found.json").read_text())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_each_recorded_file_is_written_with_its_state_and_the_states_add_up(recorded, capsys):
|
|
28
|
+
(recorded / "results.json").write_text('{"total": 7}')
|
|
29
|
+
(recorded / "inputs.json").unlink()
|
|
30
|
+
code, account = found(recorded, "--files")
|
|
31
|
+
printed = capsys.readouterr().out
|
|
32
|
+
assert code == 1 == account["exit"]
|
|
33
|
+
assert account["chain"] == "intact" and account["events"] == 3
|
|
34
|
+
assert account["files"] == [
|
|
35
|
+
{"path": "inputs.json", "state": "missing"},
|
|
36
|
+
{"path": "results.json", "state": "changed"},
|
|
37
|
+
]
|
|
38
|
+
assert "MISSING inputs.json" in printed and "CHANGED results.json" in printed
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_unchanged_files_are_ok_and_without_files_none_is_compared(recorded):
|
|
42
|
+
code, account = found(recorded, "--files")
|
|
43
|
+
assert code == 0
|
|
44
|
+
assert [file["state"] for file in account["files"]] == ["ok", "ok"]
|
|
45
|
+
code, account = found(recorded)
|
|
46
|
+
assert code == 0 and account["files"] == [] and account["files_checked"] is False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_a_chain_that_does_not_verify_is_named_and_no_file_is_compared(recorded):
|
|
50
|
+
ledger = recorded / ".results" / "ledger.jsonl"
|
|
51
|
+
ledger.write_text(ledger.read_text().replace("r1", "r2"))
|
|
52
|
+
code, account = found(recorded, "--files")
|
|
53
|
+
assert code == 1
|
|
54
|
+
assert account["chain"] == "edited" and account["problems"]
|
|
55
|
+
assert account["files"] == [] and "events" not in account
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|