sciproof 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sciproof-0.1.0/LICENSE +21 -0
- sciproof-0.1.0/MANIFEST.in +2 -0
- sciproof-0.1.0/PKG-INFO +218 -0
- sciproof-0.1.0/README.md +192 -0
- sciproof-0.1.0/pyproject.toml +53 -0
- sciproof-0.1.0/setup.cfg +4 -0
- sciproof-0.1.0/src/sciproof/__init__.py +7 -0
- sciproof-0.1.0/src/sciproof/__main__.py +6 -0
- sciproof-0.1.0/src/sciproof/capture.py +263 -0
- sciproof-0.1.0/src/sciproof/cli.py +249 -0
- sciproof-0.1.0/src/sciproof/environment.py +123 -0
- sciproof-0.1.0/src/sciproof/errors.py +25 -0
- sciproof-0.1.0/src/sciproof/gitinfo.py +42 -0
- sciproof-0.1.0/src/sciproof/hashing.py +23 -0
- sciproof-0.1.0/src/sciproof/inspect_manifest.py +98 -0
- sciproof-0.1.0/src/sciproof/manifest.py +216 -0
- sciproof-0.1.0/src/sciproof/numeric.py +189 -0
- sciproof-0.1.0/src/sciproof/paths.py +61 -0
- sciproof-0.1.0/src/sciproof/runner.py +72 -0
- sciproof-0.1.0/src/sciproof/textutil.py +21 -0
- sciproof-0.1.0/src/sciproof/verify.py +549 -0
- sciproof-0.1.0/src/sciproof.egg-info/PKG-INFO +218 -0
- sciproof-0.1.0/src/sciproof.egg-info/SOURCES.txt +33 -0
- sciproof-0.1.0/src/sciproof.egg-info/dependency_links.txt +1 -0
- sciproof-0.1.0/src/sciproof.egg-info/entry_points.txt +2 -0
- sciproof-0.1.0/src/sciproof.egg-info/requires.txt +8 -0
- sciproof-0.1.0/src/sciproof.egg-info/top_level.txt +1 -0
- sciproof-0.1.0/tests/test_capture.py +258 -0
- sciproof-0.1.0/tests/test_cli.py +195 -0
- sciproof-0.1.0/tests/test_environment_probe.py +67 -0
- sciproof-0.1.0/tests/test_manifest.py +184 -0
- sciproof-0.1.0/tests/test_numeric.py +123 -0
- sciproof-0.1.0/tests/test_numeric_npy.py +129 -0
- sciproof-0.1.0/tests/test_security.py +122 -0
- sciproof-0.1.0/tests/test_verify.py +317 -0
sciproof-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 HayagrivAI Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
sciproof-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sciproof
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Record computational experiments and create machine-verifiable evidence for checking their results.
|
|
5
|
+
Author: HayagrivAI Labs
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: reproducibility,provenance,research,verification,cli
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: numpy
|
|
20
|
+
Requires-Dist: numpy>=1.26; extra == "numpy"
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
23
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
24
|
+
Requires-Dist: numpy>=1.26; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# SciProof
|
|
28
|
+
|
|
29
|
+
[](https://github.com/hayagrivai-labs/sciproof/actions/workflows/ci.yml)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
|
|
32
|
+
Record a computational experiment once, then let anyone **check the evidence**: are the files
|
|
33
|
+
unchanged, and does re-running the command give the recorded results, exactly or within a
|
|
34
|
+
tolerance you declared?
|
|
35
|
+
|
|
36
|
+
> **Status: alpha (0.1).** Small, local-first, offline, no runtime dependencies. The idea is
|
|
37
|
+
> unproven with real users; see [the competitive analysis](docs/COMPETITIVE_ANALYSIS.md) for
|
|
38
|
+
> how it overlaps with existing tools (DVC, ReproZip, Sumatra, MLflow, RO-Crate).
|
|
39
|
+
|
|
40
|
+
## The problem
|
|
41
|
+
|
|
42
|
+
You have a script that produced a table or a figure. Months later, a reviewer, a collaborator
|
|
43
|
+
or you yourself ask: *which code and data made this, in which environment, and does it still
|
|
44
|
+
come out the same?* SciProof records the answer in a single JSON **evidence manifest** and
|
|
45
|
+
can later verify it with an explicit verdict: `PASS`, `FAIL` or `INCONCLUSIVE`.
|
|
46
|
+
|
|
47
|
+
## What it does and does not prove
|
|
48
|
+
|
|
49
|
+
| Level | Question | SciProof |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| Integrity | Are the files what was recorded? | SHA-256 checks |
|
|
52
|
+
| Execution reproducibility | Does the command still produce the recorded results? | opt-in rerun; environment *differences are reported*, not fixed |
|
|
53
|
+
| Numerical equivalence | Equal within a declared tolerance? | `--output-numeric` with `rtol`/`atol` for `.json`, `.csv` and (with NumPy) `.npy` |
|
|
54
|
+
| **Scientific validity** | **Is the result correct?** | **Not addressed. A reproducible result can still be wrong.** |
|
|
55
|
+
|
|
56
|
+
Bitwise reproducibility across different hardware or platforms is **not** promised. Details in
|
|
57
|
+
[docs/CONCEPTS.md](docs/CONCEPTS.md).
|
|
58
|
+
|
|
59
|
+
## Install
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/hayagrivai-labs/sciproof.git && cd sciproof
|
|
63
|
+
python -m pip install -e . # Python 3.11+
|
|
64
|
+
sciproof --version
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
(Not published on PyPI yet.)
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
The repository contains a small deterministic experiment (a seeded Monte Carlo estimate of pi).
|
|
72
|
+
Run everything below from the repository root, or run `python examples/demo.py` to see the whole
|
|
73
|
+
story in a temporary directory.
|
|
74
|
+
|
|
75
|
+
**1. Capture** the experiment. Declare the files that matter; everything after `--` is the
|
|
76
|
+
command (it is *not* run through a shell):
|
|
77
|
+
|
|
78
|
+
```console
|
|
79
|
+
$ sciproof capture --name baseline \
|
|
80
|
+
--input examples/experiment.py --input examples/params.json \
|
|
81
|
+
--output examples/out/results.json \
|
|
82
|
+
--output-numeric examples/out/convergence.csv --rtol 1e-9 \
|
|
83
|
+
-- python examples/experiment.py
|
|
84
|
+
Evidence written to baseline.sciproof.json
|
|
85
|
+
command exit code: 0
|
|
86
|
+
inputs recorded: 2
|
|
87
|
+
outputs recorded: 2
|
|
88
|
+
Note: stdout/stderr are stored in the manifest; check them for secrets before sharing it.
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
**2. Inspect** the evidence. This never runs the recorded command:
|
|
92
|
+
|
|
93
|
+
```console
|
|
94
|
+
$ sciproof inspect baseline.sciproof.json
|
|
95
|
+
SciProof manifest: baseline
|
|
96
|
+
format: sciproof.manifest 1.0
|
|
97
|
+
captured: 2026-10-17T18:47:12+00:00
|
|
98
|
+
|
|
99
|
+
Command (recorded, NOT executed by 'inspect'):
|
|
100
|
+
python examples/experiment.py
|
|
101
|
+
...
|
|
102
|
+
Outputs (2):
|
|
103
|
+
examples/out/results.json sha256:723d792e1b1ad975... 69 bytes compare: exact hash
|
|
104
|
+
examples/out/convergence.csv sha256:131eb2551a942bb9... 89 bytes compare: numeric (rtol=1e-09, atol=0.0)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**3. Verify integrity** (nothing is executed):
|
|
108
|
+
|
|
109
|
+
```console
|
|
110
|
+
$ sciproof verify baseline.sciproof.json
|
|
111
|
+
[PASS] input examples/experiment.py: unchanged (SHA-256 matches)
|
|
112
|
+
[PASS] input examples/params.json: unchanged (SHA-256 matches)
|
|
113
|
+
[PASS] output examples/out/results.json: unchanged (SHA-256 matches)
|
|
114
|
+
[PASS] output examples/out/convergence.csv: unchanged (SHA-256 matches)
|
|
115
|
+
|
|
116
|
+
note: scope: artifact integrity only. The recorded command was NOT executed, so this does not show that the results can be reproduced.
|
|
117
|
+
|
|
118
|
+
RESULT: PASS
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
**4. Verify by re-execution.** Running the command is a deliberate act: it needs `--rerun` and
|
|
122
|
+
either an interactive confirmation or `--yes`.
|
|
123
|
+
|
|
124
|
+
```console
|
|
125
|
+
$ sciproof verify baseline.sciproof.json --rerun --yes
|
|
126
|
+
...
|
|
127
|
+
[PASS] exit code: 0, same as recorded
|
|
128
|
+
[PASS] rerun output examples/out/results.json: identical to the recorded output (exact SHA-256)
|
|
129
|
+
[PASS] rerun output examples/out/convergence.csv: numerically equivalent (rtol=1e-09, atol=0.0)
|
|
130
|
+
RESULT: PASS
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
**5. A modified output is detected:**
|
|
134
|
+
|
|
135
|
+
```console
|
|
136
|
+
$ sciproof verify baseline.sciproof.json
|
|
137
|
+
[FAIL] output examples/out/results.json: modified since capture (SHA-256 differs)
|
|
138
|
+
RESULT: FAIL
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Exit codes: `0` PASS, `1` FAIL, `2` INCONCLUSIVE, `3` usage error or invalid manifest. A missing
|
|
142
|
+
input, a missing numeric snapshot, a command that cannot be started, a timeout or a declined
|
|
143
|
+
rerun are `INCONCLUSIVE`, never `PASS`.
|
|
144
|
+
|
|
145
|
+
A walkthrough with NumPy and pandas, including how to choose tolerances: [docs/TUTORIAL_NUMPY_PANDAS.md](docs/TUTORIAL_NUMPY_PANDAS.md).
|
|
146
|
+
|
|
147
|
+
## Commands
|
|
148
|
+
|
|
149
|
+
```text
|
|
150
|
+
sciproof capture --name NAME [--input P]... [--output P]... [--output-numeric P]...
|
|
151
|
+
[--rtol X] [--atol Y] [--timeout S] [--max-log-bytes N]
|
|
152
|
+
[--force] [--allow-nonzero] -- COMMAND [ARGS...]
|
|
153
|
+
sciproof verify MANIFEST [--rerun] [--yes] [--strict-env] [--timeout S] [--json]
|
|
154
|
+
sciproof inspect MANIFEST
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
* Paths must be inside the current directory; the manifest is written there as
|
|
158
|
+
`NAME.sciproof.json`. Numeric outputs are also copied to `NAME.sciproof.d/` (needed to compare
|
|
159
|
+
with a tolerance; keep it next to the manifest).
|
|
160
|
+
* Declared inputs must be read-only for the command; capture fails if the command changes one.
|
|
161
|
+
* A command that exits with a nonzero status is still recorded (capture then exits with `1`
|
|
162
|
+
unless `--allow-nonzero`).
|
|
163
|
+
* `--json` prints the verification report as a single JSON object (same exit codes), for CI
|
|
164
|
+
and scripts. Prompts and errors go to stderr, so stdout stays valid JSON.
|
|
165
|
+
* `--strict-env` makes a rerun `INCONCLUSIVE` (without executing) if Python version, platform
|
|
166
|
+
or recorded package versions differ.
|
|
167
|
+
|
|
168
|
+
Manifest format: [docs/MANIFEST.md](docs/MANIFEST.md).
|
|
169
|
+
|
|
170
|
+
## Threat model and limitations
|
|
171
|
+
|
|
172
|
+
* A manifest contains a command and must be treated as untrusted. `inspect` and plain `verify`
|
|
173
|
+
never execute it; `--rerun` runs it **with your permissions, unsandboxed**. Only rerun
|
|
174
|
+
manifests you trust, ideally in a disposable VM or container. See [SECURITY.md](SECURITY.md).
|
|
175
|
+
* Manifests are **not signed** (a tested recipe using `ssh-keygen -Y` is in
|
|
176
|
+
[docs/DESIGN_SIGNING.md](docs/DESIGN_SIGNING.md)): hashes detect changes, they do not stop someone who can rewrite
|
|
177
|
+
both manifest and files from fabricating consistent evidence.
|
|
178
|
+
* Only declared files are tracked. Undeclared inputs, network, time and system libraries are
|
|
179
|
+
invisible. SciProof reports environment differences but does not recreate environments.
|
|
180
|
+
* stdout/stderr and the command line are stored in the manifest; review them for secrets.
|
|
181
|
+
* Numeric comparison supports `.json` and `.csv` (standard library only) and `.npy` arrays
|
|
182
|
+
(`pip install 'sciproof[numpy]'`; numeric, boolean and complex dtypes, never unpickled).
|
|
183
|
+
* The environment is probed from the command's interpreter only if the command is a Python
|
|
184
|
+
interpreter; otherwise the interpreter running SciProof is described (and the manifest says so).
|
|
185
|
+
* Linux, Windows and macOS are tested in CI (Python 3.11-3.13).
|
|
186
|
+
* The unit tests that check symlink escapes need symlink support (they run in Linux CI and are
|
|
187
|
+
skipped where symlinks are unavailable).
|
|
188
|
+
|
|
189
|
+
## Related tools
|
|
190
|
+
|
|
191
|
+
| Tool | Relation |
|
|
192
|
+
| --- | --- |
|
|
193
|
+
| [DVC](https://doc.dvc.org/start/data-pipelines/data-pipelines) | Pipelines with hashes of dependencies and outputs; heavier, more powerful. If you already use it you largely have integrity checking. |
|
|
194
|
+
| [ReproZip](https://docs.reprozip.org/en/1.x/) | Packs a Linux experiment with its files and dependencies so it can be rerun elsewhere. SciProof does not package anything. |
|
|
195
|
+
| [Sumatra](https://sumatra.readthedocs.io/en/latest/introduction.html) | Electronic-lab-notebook style run records. |
|
|
196
|
+
| [MLflow Tracking](https://mlflow.org/docs/latest/ml/tracking/) | Experiment tracking for ML with parameters and metrics, optional UI. |
|
|
197
|
+
| [RO-Crate](https://www.researchobject.org/ro-crate/specification/1.1/introduction.html) | A standard to *describe* research outputs; does not verify them. Mapping study: [docs/DESIGN_RO_CRATE.md](docs/DESIGN_RO_CRATE.md). |
|
|
198
|
+
|
|
199
|
+
Full analysis, with what could not be confirmed: [docs/COMPETITIVE_ANALYSIS.md](docs/COMPETITIVE_ANALYSIS.md).
|
|
200
|
+
|
|
201
|
+
## Roadmap (decided by feedback, not by speculation)
|
|
202
|
+
|
|
203
|
+
Nothing below is promised. Each item is only worth doing if real users ask for it:
|
|
204
|
+
|
|
205
|
+
1. Try it on real experiments of a few researchers; record what is awkward or missing.
|
|
206
|
+
2. Based on that: more numeric formats (for example `.npz`), directory outputs, or a manifest export to RO-Crate
|
|
207
|
+
for interoperability.
|
|
208
|
+
3. Signing or timestamping of manifests, if integrity against a hostile party matters to users.
|
|
209
|
+
|
|
210
|
+
## Contributing
|
|
211
|
+
|
|
212
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Quick checks: `ruff check .`, `ruff format --check .`,
|
|
213
|
+
`python -m pytest`, `python examples/demo.py`.
|
|
214
|
+
|
|
215
|
+
## Licence
|
|
216
|
+
|
|
217
|
+
[MIT](LICENSE): short and permissive, so researchers and companies can use and embed it
|
|
218
|
+
without legal review.
|
sciproof-0.1.0/README.md
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# SciProof
|
|
2
|
+
|
|
3
|
+
[](https://github.com/hayagrivai-labs/sciproof/actions/workflows/ci.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
6
|
+
Record a computational experiment once, then let anyone **check the evidence**: are the files
|
|
7
|
+
unchanged, and does re-running the command give the recorded results, exactly or within a
|
|
8
|
+
tolerance you declared?
|
|
9
|
+
|
|
10
|
+
> **Status: alpha (0.1).** Small, local-first, offline, no runtime dependencies. The idea is
|
|
11
|
+
> unproven with real users; see [the competitive analysis](docs/COMPETITIVE_ANALYSIS.md) for
|
|
12
|
+
> how it overlaps with existing tools (DVC, ReproZip, Sumatra, MLflow, RO-Crate).
|
|
13
|
+
|
|
14
|
+
## The problem
|
|
15
|
+
|
|
16
|
+
You have a script that produced a table or a figure. Months later, a reviewer, a collaborator
|
|
17
|
+
or you yourself ask: *which code and data made this, in which environment, and does it still
|
|
18
|
+
come out the same?* SciProof records the answer in a single JSON **evidence manifest** and
|
|
19
|
+
can later verify it with an explicit verdict: `PASS`, `FAIL` or `INCONCLUSIVE`.
|
|
20
|
+
|
|
21
|
+
## What it does and does not prove
|
|
22
|
+
|
|
23
|
+
| Level | Question | SciProof |
|
|
24
|
+
| --- | --- | --- |
|
|
25
|
+
| Integrity | Are the files what was recorded? | SHA-256 checks |
|
|
26
|
+
| Execution reproducibility | Does the command still produce the recorded results? | opt-in rerun; environment *differences are reported*, not fixed |
|
|
27
|
+
| Numerical equivalence | Equal within a declared tolerance? | `--output-numeric` with `rtol`/`atol` for `.json`, `.csv` and (with NumPy) `.npy` |
|
|
28
|
+
| **Scientific validity** | **Is the result correct?** | **Not addressed. A reproducible result can still be wrong.** |
|
|
29
|
+
|
|
30
|
+
Bitwise reproducibility across different hardware or platforms is **not** promised. Details in
|
|
31
|
+
[docs/CONCEPTS.md](docs/CONCEPTS.md).
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
git clone https://github.com/hayagrivai-labs/sciproof.git && cd sciproof
|
|
37
|
+
python -m pip install -e . # Python 3.11+
|
|
38
|
+
sciproof --version
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
(Not published on PyPI yet.)
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
The repository contains a small deterministic experiment (a seeded Monte Carlo estimate of pi).
|
|
46
|
+
Run everything below from the repository root, or run `python examples/demo.py` to see the whole
|
|
47
|
+
story in a temporary directory.
|
|
48
|
+
|
|
49
|
+
**1. Capture** the experiment. Declare the files that matter; everything after `--` is the
|
|
50
|
+
command (it is *not* run through a shell):
|
|
51
|
+
|
|
52
|
+
```console
|
|
53
|
+
$ sciproof capture --name baseline \
|
|
54
|
+
--input examples/experiment.py --input examples/params.json \
|
|
55
|
+
--output examples/out/results.json \
|
|
56
|
+
--output-numeric examples/out/convergence.csv --rtol 1e-9 \
|
|
57
|
+
-- python examples/experiment.py
|
|
58
|
+
Evidence written to baseline.sciproof.json
|
|
59
|
+
command exit code: 0
|
|
60
|
+
inputs recorded: 2
|
|
61
|
+
outputs recorded: 2
|
|
62
|
+
Note: stdout/stderr are stored in the manifest; check them for secrets before sharing it.
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**2. Inspect** the evidence. This never runs the recorded command:
|
|
66
|
+
|
|
67
|
+
```console
|
|
68
|
+
$ sciproof inspect baseline.sciproof.json
|
|
69
|
+
SciProof manifest: baseline
|
|
70
|
+
format: sciproof.manifest 1.0
|
|
71
|
+
captured: 2026-10-17T18:47:12+00:00
|
|
72
|
+
|
|
73
|
+
Command (recorded, NOT executed by 'inspect'):
|
|
74
|
+
python examples/experiment.py
|
|
75
|
+
...
|
|
76
|
+
Outputs (2):
|
|
77
|
+
examples/out/results.json sha256:723d792e1b1ad975... 69 bytes compare: exact hash
|
|
78
|
+
examples/out/convergence.csv sha256:131eb2551a942bb9... 89 bytes compare: numeric (rtol=1e-09, atol=0.0)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**3. Verify integrity** (nothing is executed):
|
|
82
|
+
|
|
83
|
+
```console
|
|
84
|
+
$ sciproof verify baseline.sciproof.json
|
|
85
|
+
[PASS] input examples/experiment.py: unchanged (SHA-256 matches)
|
|
86
|
+
[PASS] input examples/params.json: unchanged (SHA-256 matches)
|
|
87
|
+
[PASS] output examples/out/results.json: unchanged (SHA-256 matches)
|
|
88
|
+
[PASS] output examples/out/convergence.csv: unchanged (SHA-256 matches)
|
|
89
|
+
|
|
90
|
+
note: scope: artifact integrity only. The recorded command was NOT executed, so this does not show that the results can be reproduced.
|
|
91
|
+
|
|
92
|
+
RESULT: PASS
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**4. Verify by re-execution.** Running the command is a deliberate act: it needs `--rerun` and
|
|
96
|
+
either an interactive confirmation or `--yes`.
|
|
97
|
+
|
|
98
|
+
```console
|
|
99
|
+
$ sciproof verify baseline.sciproof.json --rerun --yes
|
|
100
|
+
...
|
|
101
|
+
[PASS] exit code: 0, same as recorded
|
|
102
|
+
[PASS] rerun output examples/out/results.json: identical to the recorded output (exact SHA-256)
|
|
103
|
+
[PASS] rerun output examples/out/convergence.csv: numerically equivalent (rtol=1e-09, atol=0.0)
|
|
104
|
+
RESULT: PASS
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**5. A modified output is detected:**
|
|
108
|
+
|
|
109
|
+
```console
|
|
110
|
+
$ sciproof verify baseline.sciproof.json
|
|
111
|
+
[FAIL] output examples/out/results.json: modified since capture (SHA-256 differs)
|
|
112
|
+
RESULT: FAIL
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Exit codes: `0` PASS, `1` FAIL, `2` INCONCLUSIVE, `3` usage error or invalid manifest. A missing
|
|
116
|
+
input, a missing numeric snapshot, a command that cannot be started, a timeout or a declined
|
|
117
|
+
rerun are `INCONCLUSIVE`, never `PASS`.
|
|
118
|
+
|
|
119
|
+
A walkthrough with NumPy and pandas, including how to choose tolerances: [docs/TUTORIAL_NUMPY_PANDAS.md](docs/TUTORIAL_NUMPY_PANDAS.md).
|
|
120
|
+
|
|
121
|
+
## Commands
|
|
122
|
+
|
|
123
|
+
```text
|
|
124
|
+
sciproof capture --name NAME [--input P]... [--output P]... [--output-numeric P]...
|
|
125
|
+
[--rtol X] [--atol Y] [--timeout S] [--max-log-bytes N]
|
|
126
|
+
[--force] [--allow-nonzero] -- COMMAND [ARGS...]
|
|
127
|
+
sciproof verify MANIFEST [--rerun] [--yes] [--strict-env] [--timeout S] [--json]
|
|
128
|
+
sciproof inspect MANIFEST
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
* Paths must be inside the current directory; the manifest is written there as
|
|
132
|
+
`NAME.sciproof.json`. Numeric outputs are also copied to `NAME.sciproof.d/` (needed to compare
|
|
133
|
+
with a tolerance; keep it next to the manifest).
|
|
134
|
+
* Declared inputs must be read-only for the command; capture fails if the command changes one.
|
|
135
|
+
* A command that exits with a nonzero status is still recorded (capture then exits with `1`
|
|
136
|
+
unless `--allow-nonzero`).
|
|
137
|
+
* `--json` prints the verification report as a single JSON object (same exit codes), for CI
|
|
138
|
+
and scripts. Prompts and errors go to stderr, so stdout stays valid JSON.
|
|
139
|
+
* `--strict-env` makes a rerun `INCONCLUSIVE` (without executing) if Python version, platform
|
|
140
|
+
or recorded package versions differ.
|
|
141
|
+
|
|
142
|
+
Manifest format: [docs/MANIFEST.md](docs/MANIFEST.md).
|
|
143
|
+
|
|
144
|
+
## Threat model and limitations
|
|
145
|
+
|
|
146
|
+
* A manifest contains a command and must be treated as untrusted. `inspect` and plain `verify`
|
|
147
|
+
never execute it; `--rerun` runs it **with your permissions, unsandboxed**. Only rerun
|
|
148
|
+
manifests you trust, ideally in a disposable VM or container. See [SECURITY.md](SECURITY.md).
|
|
149
|
+
* Manifests are **not signed** (a tested recipe using `ssh-keygen -Y` is in
|
|
150
|
+
[docs/DESIGN_SIGNING.md](docs/DESIGN_SIGNING.md)): hashes detect changes, they do not stop someone who can rewrite
|
|
151
|
+
both manifest and files from fabricating consistent evidence.
|
|
152
|
+
* Only declared files are tracked. Undeclared inputs, network, time and system libraries are
|
|
153
|
+
invisible. SciProof reports environment differences but does not recreate environments.
|
|
154
|
+
* stdout/stderr and the command line are stored in the manifest; review them for secrets.
|
|
155
|
+
* Numeric comparison supports `.json` and `.csv` (standard library only) and `.npy` arrays
|
|
156
|
+
(`pip install 'sciproof[numpy]'`; numeric, boolean and complex dtypes, never unpickled).
|
|
157
|
+
* The environment is probed from the command's interpreter only if the command is a Python
|
|
158
|
+
interpreter; otherwise the interpreter running SciProof is described (and the manifest says so).
|
|
159
|
+
* Linux, Windows and macOS are tested in CI (Python 3.11-3.13).
|
|
160
|
+
* The unit tests that check symlink escapes need symlink support (they run in Linux CI and are
|
|
161
|
+
skipped where symlinks are unavailable).
|
|
162
|
+
|
|
163
|
+
## Related tools
|
|
164
|
+
|
|
165
|
+
| Tool | Relation |
|
|
166
|
+
| --- | --- |
|
|
167
|
+
| [DVC](https://doc.dvc.org/start/data-pipelines/data-pipelines) | Pipelines with hashes of dependencies and outputs; heavier, more powerful. If you already use it you largely have integrity checking. |
|
|
168
|
+
| [ReproZip](https://docs.reprozip.org/en/1.x/) | Packs a Linux experiment with its files and dependencies so it can be rerun elsewhere. SciProof does not package anything. |
|
|
169
|
+
| [Sumatra](https://sumatra.readthedocs.io/en/latest/introduction.html) | Electronic-lab-notebook style run records. |
|
|
170
|
+
| [MLflow Tracking](https://mlflow.org/docs/latest/ml/tracking/) | Experiment tracking for ML with parameters and metrics, optional UI. |
|
|
171
|
+
| [RO-Crate](https://www.researchobject.org/ro-crate/specification/1.1/introduction.html) | A standard to *describe* research outputs; does not verify them. Mapping study: [docs/DESIGN_RO_CRATE.md](docs/DESIGN_RO_CRATE.md). |
|
|
172
|
+
|
|
173
|
+
Full analysis, with what could not be confirmed: [docs/COMPETITIVE_ANALYSIS.md](docs/COMPETITIVE_ANALYSIS.md).
|
|
174
|
+
|
|
175
|
+
## Roadmap (decided by feedback, not by speculation)
|
|
176
|
+
|
|
177
|
+
Nothing below is promised. Each item is only worth doing if real users ask for it:
|
|
178
|
+
|
|
179
|
+
1. Try it on real experiments of a few researchers; record what is awkward or missing.
|
|
180
|
+
2. Based on that: more numeric formats (for example `.npz`), directory outputs, or a manifest export to RO-Crate
|
|
181
|
+
for interoperability.
|
|
182
|
+
3. Signing or timestamping of manifests, if integrity against a hostile party matters to users.
|
|
183
|
+
|
|
184
|
+
## Contributing
|
|
185
|
+
|
|
186
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Quick checks: `ruff check .`, `ruff format --check .`,
|
|
187
|
+
`python -m pytest`, `python examples/demo.py`.
|
|
188
|
+
|
|
189
|
+
## Licence
|
|
190
|
+
|
|
191
|
+
[MIT](LICENSE): short and permissive, so researchers and companies can use and embed it
|
|
192
|
+
without legal review.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sciproof"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Record computational experiments and create machine-verifiable evidence for checking their results."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "HayagrivAI Labs" }]
|
|
14
|
+
keywords = ["reproducibility", "provenance", "research", "verification", "cli"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Scientific/Engineering",
|
|
24
|
+
]
|
|
25
|
+
# No runtime dependencies on purpose: the evidence tool should be easy to audit.
|
|
26
|
+
dependencies = []
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
numpy = ["numpy>=1.26"]
|
|
30
|
+
dev = ["pytest>=8", "ruff>=0.5", "numpy>=1.26"]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
sciproof = "sciproof.cli:main"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
testpaths = ["tests"]
|
|
40
|
+
addopts = "-ra"
|
|
41
|
+
|
|
42
|
+
[tool.ruff]
|
|
43
|
+
line-length = 100
|
|
44
|
+
target-version = "py311"
|
|
45
|
+
src = ["src", "tests"]
|
|
46
|
+
|
|
47
|
+
[tool.ruff.lint]
|
|
48
|
+
select = ["E", "F", "W", "I", "B", "UP", "S", "SIM", "C4", "PIE", "RET"]
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint.per-file-ignores]
|
|
51
|
+
# Tests deliberately use assert and run subprocesses with fixed arguments.
|
|
52
|
+
"tests/*" = ["S101", "S603", "S607", "S105", "S106"]
|
|
53
|
+
"examples/*" = ["S311"]
|
sciproof-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""SciProof: record computational experiments and create verifiable evidence for them.
|
|
2
|
+
|
|
3
|
+
SciProof checks *integrity* and *execution reproducibility* of recorded runs. It does not,
|
|
4
|
+
and cannot, establish that a scientific result is correct. See docs/CONCEPTS.md.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|