notedelta 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- notedelta-0.1.0/.gitignore +33 -0
- notedelta-0.1.0/CHANGELOG.md +39 -0
- notedelta-0.1.0/LICENSE +43 -0
- notedelta-0.1.0/PKG-INFO +187 -0
- notedelta-0.1.0/README.md +143 -0
- notedelta-0.1.0/THIRD_PARTY_NOTICES.md +24 -0
- notedelta-0.1.0/extensions/vscode/CHANGELOG.md +7 -0
- notedelta-0.1.0/extensions/vscode/LICENSE +43 -0
- notedelta-0.1.0/extensions/vscode/README.md +51 -0
- notedelta-0.1.0/public/README.md +146 -0
- notedelta-0.1.0/pyproject.toml +105 -0
- notedelta-0.1.0/src/notedelta/__init__.py +15 -0
- notedelta-0.1.0/src/notedelta/__main__.py +3 -0
- notedelta-0.1.0/src/notedelta/_version.py +1 -0
- notedelta-0.1.0/src/notedelta/cli.py +473 -0
- notedelta-0.1.0/src/notedelta/config/__init__.py +4 -0
- notedelta-0.1.0/src/notedelta/config/loader.py +137 -0
- notedelta-0.1.0/src/notedelta/config/schema.py +326 -0
- notedelta-0.1.0/src/notedelta/engine.py +761 -0
- notedelta-0.1.0/src/notedelta/environment/__init__.py +0 -0
- notedelta-0.1.0/src/notedelta/environment/deps.py +503 -0
- notedelta-0.1.0/src/notedelta/environment/notebook_env.py +183 -0
- notedelta-0.1.0/src/notedelta/environment/seeds.py +175 -0
- notedelta-0.1.0/src/notedelta/errors.py +44 -0
- notedelta-0.1.0/src/notedelta/execution/__init__.py +0 -0
- notedelta-0.1.0/src/notedelta/execution/runner.py +143 -0
- notedelta-0.1.0/src/notedelta/gitref.py +127 -0
- notedelta-0.1.0/src/notedelta/limits.py +29 -0
- notedelta-0.1.0/src/notedelta/matching/__init__.py +3 -0
- notedelta-0.1.0/src/notedelta/matching/cells.py +284 -0
- notedelta-0.1.0/src/notedelta/metrics/__init__.py +0 -0
- notedelta-0.1.0/src/notedelta/metrics/extract.py +160 -0
- notedelta-0.1.0/src/notedelta/model.py +300 -0
- notedelta-0.1.0/src/notedelta/notebooks/__init__.py +4 -0
- notedelta-0.1.0/src/notedelta/notebooks/loader.py +259 -0
- notedelta-0.1.0/src/notedelta/notebooks/model.py +179 -0
- notedelta-0.1.0/src/notedelta/numeric/__init__.py +0 -0
- notedelta-0.1.0/src/notedelta/numeric/compare.py +227 -0
- notedelta-0.1.0/src/notedelta/numeric/discover.py +181 -0
- notedelta-0.1.0/src/notedelta/numeric/literal.py +456 -0
- notedelta-0.1.0/src/notedelta/numeric/numbers.py +183 -0
- notedelta-0.1.0/src/notedelta/numeric/table.py +264 -0
- notedelta-0.1.0/src/notedelta/numeric/text.py +190 -0
- notedelta-0.1.0/src/notedelta/regression/__init__.py +0 -0
- notedelta-0.1.0/src/notedelta/regression/classify.py +275 -0
- notedelta-0.1.0/src/notedelta/reporting/__init__.py +11 -0
- notedelta-0.1.0/src/notedelta/reporting/common.py +144 -0
- notedelta-0.1.0/src/notedelta/reporting/html.py +503 -0
- notedelta-0.1.0/src/notedelta/reporting/markdown.py +104 -0
- notedelta-0.1.0/src/notedelta/reporting/redact.py +70 -0
- notedelta-0.1.0/src/notedelta/reporting/terminal.py +170 -0
- notedelta-0.1.0/src/notedelta/stability.py +111 -0
- notedelta-0.1.0/tests/conftest.py +39 -0
- notedelta-0.1.0/tests/fixtures/build_fixtures.py +496 -0
- notedelta-0.1.0/tests/fixtures/edge/corrupt.ipynb +1 -0
- notedelta-0.1.0/tests/fixtures/edge/empty.ipynb +0 -0
- notedelta-0.1.0/tests/fixtures/edge/hostile_a.ipynb +85 -0
- notedelta-0.1.0/tests/fixtures/edge/hostile_b.ipynb +86 -0
- notedelta-0.1.0/tests/fixtures/edge/list.ipynb +1 -0
- notedelta-0.1.0/tests/fixtures/edge/no_ids.ipynb +49 -0
- notedelta-0.1.0/tests/fixtures/edge/not_a_notebook.ipynb +1 -0
- notedelta-0.1.0/tests/fixtures/edge/v3.ipynb +31 -0
- notedelta-0.1.0/tests/fixtures/scenario_a_ml/.notedelta.yml +21 -0
- notedelta-0.1.0/tests/fixtures/scenario_a_ml/baseline.ipynb +220 -0
- notedelta-0.1.0/tests/fixtures/scenario_a_ml/candidate.ipynb +220 -0
- notedelta-0.1.0/tests/fixtures/scenario_b_simulation/.notedelta.yml +33 -0
- notedelta-0.1.0/tests/fixtures/scenario_b_simulation/baseline.ipynb +150 -0
- notedelta-0.1.0/tests/fixtures/scenario_b_simulation/candidate.ipynb +150 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/.notedelta-root +1 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/analysis.ipynb +90 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/requirements.txt +3 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/uv.lock +14 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/.notedelta-root +1 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/analysis.ipynb +90 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/requirements.txt +3 -0
- notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/uv.lock +14 -0
- notedelta-0.1.0/tests/integration/test_cli.py +147 -0
- notedelta-0.1.0/tests/integration/test_execute.py +95 -0
- notedelta-0.1.0/tests/integration/test_git.py +104 -0
- notedelta-0.1.0/tests/integration/test_github_action.py +167 -0
- notedelta-0.1.0/tests/integration/test_scenarios.py +130 -0
- notedelta-0.1.0/tests/regression/test_report_schema.py +67 -0
- notedelta-0.1.0/tests/unit/test_classify_and_extract.py +192 -0
- notedelta-0.1.0/tests/unit/test_compare.py +175 -0
- notedelta-0.1.0/tests/unit/test_config.py +137 -0
- notedelta-0.1.0/tests/unit/test_environment.py +161 -0
- notedelta-0.1.0/tests/unit/test_literal_and_text.py +109 -0
- notedelta-0.1.0/tests/unit/test_loader.py +136 -0
- notedelta-0.1.0/tests/unit/test_matching.py +160 -0
- notedelta-0.1.0/tests/unit/test_numbers.py +98 -0
- notedelta-0.1.0/tests/unit/test_report_safety.py +92 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
.venv/
|
|
3
|
+
venv/
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.py[cod]
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.coverage
|
|
11
|
+
coverage.xml
|
|
12
|
+
htmlcov/
|
|
13
|
+
build/
|
|
14
|
+
dist/
|
|
15
|
+
|
|
16
|
+
# NoteDelta outputs
|
|
17
|
+
notedelta-runs/
|
|
18
|
+
notedelta-reports/
|
|
19
|
+
*.notedelta.html
|
|
20
|
+
/report*.html
|
|
21
|
+
/report*.json
|
|
22
|
+
|
|
23
|
+
# Node / VS Code extension
|
|
24
|
+
node_modules/
|
|
25
|
+
extensions/vscode/out/
|
|
26
|
+
extensions/vscode/*.vsix
|
|
27
|
+
.vscode-test/
|
|
28
|
+
|
|
29
|
+
# Editors / OS
|
|
30
|
+
.idea/
|
|
31
|
+
.DS_Store
|
|
32
|
+
Thumbs.db
|
|
33
|
+
.ipynb_checkpoints/
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes are listed here. Versions follow semantic versioning; the JSON
|
|
4
|
+
report and the configuration carry their own schema versions.
|
|
5
|
+
|
|
6
|
+
## 0.1.0 - 2026-11-15 (first beta)
|
|
7
|
+
|
|
8
|
+
- Renamed from the working name LabTrace to NoteDelta before the first release (the
|
|
9
|
+
`labtrace` name was taken on PyPI). Package, command, configuration file
|
|
10
|
+
(`.notedelta.yml`) and VS Code identifiers all use `notedelta`.
|
|
11
|
+
|
|
12
|
+
Core and CLI
|
|
13
|
+
- `notedelta compare`: cell matching by stable ids, identical content and similarity,
|
|
14
|
+
with reorder detection and explicit ambiguity flags; source, output, error, execution
|
|
15
|
+
count and metadata diffs; image outputs compared by hash.
|
|
16
|
+
- Numerical comparison with exact decimal arithmetic, absolute/relative tolerances,
|
|
17
|
+
careful handling of zero baselines, negative values, NaN, infinities and display
|
|
18
|
+
precision; arrays (shape check, elementwise, max/mean/RMS/relative L2) and pandas
|
|
19
|
+
tables.
|
|
20
|
+
- Automatic discovery of numbers in logs, printed values, NumPy arrays, JSON outputs
|
|
21
|
+
and tables, via template-aligned text and a safe literal parser.
|
|
22
|
+
- `.notedelta.yml` v1: configured metrics (regex, key/value, JSON path extraction),
|
|
23
|
+
directions, units, tolerances, cell selectors, discovery and policy settings; strict
|
|
24
|
+
validation and regex safety checks.
|
|
25
|
+
- Explainable classification (regression, improvement, change, within tolerance,
|
|
26
|
+
missing, incomparable).
|
|
27
|
+
- Environment diagnostics: kernel, Python version, interpreter, execution order, seeds,
|
|
28
|
+
requirements/pyproject/uv/poetry/Pipfile/conda files, graded as fact/warning/hypothesis.
|
|
29
|
+
- Reports: terminal, JSON (schema 1.0, JSON Schema via `notedelta schema`), Markdown,
|
|
30
|
+
self-contained script-free HTML with light/dark themes. Best-effort redaction.
|
|
31
|
+
- `notedelta compare-git`, `check-config`, `init`, `schema`.
|
|
32
|
+
- Opt-in `notedelta execute` (nbclient; timeouts, repeats, provenance; not sandboxed) and
|
|
33
|
+
`notedelta stability`.
|
|
34
|
+
|
|
35
|
+
Integrations
|
|
36
|
+
- VS Code extension (preview): compare with git baseline or file, results view,
|
|
37
|
+
report panel, jump to cell, regression/reproducibility quick picks, starter config.
|
|
38
|
+
- GitHub composite action with job summary, report artifact and least-privilege example
|
|
39
|
+
workflow.
|
notedelta-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
NoteDelta End User License Agreement
|
|
2
|
+
Version 1.0 (draft for the 0.1 beta)
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2026 Ouyang Matters. All rights reserved.
|
|
5
|
+
|
|
6
|
+
This agreement covers the NoteDelta command-line tool and Python package, the NoteDelta
|
|
7
|
+
extension for Visual Studio Code, the NoteDelta GitHub Actions integration and their
|
|
8
|
+
documentation (together, the "Software").
|
|
9
|
+
|
|
10
|
+
1. Licence grant. You may install and use the Software, free of charge, on any number
|
|
11
|
+
of computers, for personal, academic, research and commercial purposes, including
|
|
12
|
+
in continuous-integration systems that you or your organisation operate.
|
|
13
|
+
|
|
14
|
+
2. Restrictions. You may not (a) sell, rent or sublicense the Software, (b) distribute
|
|
15
|
+
modified copies of the Software, or (c) remove or alter copyright or licence notices.
|
|
16
|
+
You may redistribute unmodified copies of the Software, including this agreement,
|
|
17
|
+
for example as part of an internal mirror or a container image.
|
|
18
|
+
|
|
19
|
+
3. Your data. The Software processes your notebooks and project files locally. It
|
|
20
|
+
does not collect telemetry and does not transmit your notebooks, results or source
|
|
21
|
+
code anywhere. Reports it writes stay where you put them.
|
|
22
|
+
|
|
23
|
+
4. Future editions. Some future features may be offered under a separate paid
|
|
24
|
+
licence. Features available free of charge in a released version will not be
|
|
25
|
+
removed from that version, and the Software contains no hidden time limits or
|
|
26
|
+
disabling mechanisms.
|
|
27
|
+
|
|
28
|
+
5. Third-party components. The Software depends on open-source packages that are
|
|
29
|
+
licensed under their own terms (see THIRD_PARTY_NOTICES.md). Nothing in this
|
|
30
|
+
agreement limits your rights under those licences.
|
|
31
|
+
|
|
32
|
+
6. No warranty. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
33
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
34
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. NoteDelta
|
|
35
|
+
reports observable differences between notebooks according to documented rules;
|
|
36
|
+
it does not determine whether results are scientifically valid or reproducible.
|
|
37
|
+
|
|
38
|
+
7. Limitation of liability. TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL
|
|
39
|
+
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
40
|
+
LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR ITS USE.
|
|
41
|
+
|
|
42
|
+
8. Termination. This licence ends automatically if you breach it. Sections 6 and 7
|
|
43
|
+
survive termination.
|
notedelta-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: notedelta
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scientific notebook review: what changed in your numerical results, how much, and can you trust the comparison.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ouyang-matters/notedelta
|
|
6
|
+
Project-URL: Documentation, https://github.com/ouyang-matters/notedelta/tree/main/docs
|
|
7
|
+
Project-URL: Issues, https://github.com/ouyang-matters/notedelta/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/ouyang-matters/notedelta/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Ouyang Matters
|
|
10
|
+
License-Expression: LicenseRef-NoteDelta-EULA
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: diff,jupyter,notebook,numerical,regression,reproducibility,scientific computing
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Framework :: Jupyter
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: nbformat>=5.7
|
|
27
|
+
Requires-Dist: packaging>=23.0
|
|
28
|
+
Requires-Dist: pydantic<3,>=2.6
|
|
29
|
+
Requires-Dist: pyyaml>=6.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
32
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest-cov>=5; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
36
|
+
Requires-Dist: types-pyyaml; extra == 'dev'
|
|
37
|
+
Provides-Extra: execute
|
|
38
|
+
Requires-Dist: ipykernel>=6.0; extra == 'execute'
|
|
39
|
+
Requires-Dist: nbclient>=0.9; extra == 'execute'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# NoteDelta
|
|
43
|
+
|
|
44
|
+
**Know what changed in your scientific results, not just your code.**
|
|
45
|
+
|
|
46
|
+
NoteDelta compares two versions of a Jupyter notebook and answers the question a code diff
|
|
47
|
+
cannot: *which numerical results changed, by how much, does that exceed what I consider
|
|
48
|
+
acceptable, and is there something about the environment I should check?*
|
|
49
|
+
|
|
50
|
+
- **Configured scientific metrics.** Declare a metric once: how to find it, whether
|
|
51
|
+
lower or higher is better, and a tolerance. NoteDelta reports the exact difference,
|
|
52
|
+
the relative change, and whether it is a regression, an improvement or within
|
|
53
|
+
tolerance, and explains the rule it applied.
|
|
54
|
+
- **Automatic numerical discovery.** Numbers in changed outputs (logs, printed values,
|
|
55
|
+
NumPy arrays, pandas tables, JSON) are paired and compared without configuration.
|
|
56
|
+
Because their direction is unknown, they are reported as changes, never as regressions.
|
|
57
|
+
- **Reproducibility diagnostics.** Differences in kernel, Python version, locked package
|
|
58
|
+
versions, dependency declarations, random seeds and execution order are each labelled
|
|
59
|
+
as a *fact*, a *warning* or a *hypothesis*. NoteDelta never claims a cause it cannot show.
|
|
60
|
+
|
|
61
|
+
NoteDelta runs entirely offline, never executes notebook code during comparison, sends no
|
|
62
|
+
telemetry and needs no account.
|
|
63
|
+
|
|
64
|
+

|
|
65
|
+
|
|
66
|
+
> **Status: beta (0.1).** The VS Code extension and the GitHub Marketplace listing are in
|
|
67
|
+
> preparation. This repository hosts the documentation, the issue tracker, the licence and
|
|
68
|
+
> the GitHub Action. The source code is not public.
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
NoteDelta needs Python 3.11 or newer and runs on Windows, macOS and Linux.
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install notedelta
|
|
76
|
+
pip install "notedelta[execute]" # optional: opt-in notebook re-execution (runs code; not sandboxed)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Quick look
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
notedelta compare baseline.ipynb experiment.ipynb
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
```text
|
|
86
|
+
NoteDelta baseline.ipynb → candidate.ipynb
|
|
87
|
+
1 regression, 2 numerical changes
|
|
88
|
+
|
|
89
|
+
Scientific metrics
|
|
90
|
+
Metric Baseline Candidate Difference Tolerance Status
|
|
91
|
+
Relative reconstruction error 0.0073 0.0091 +0.0018 (+24.66%) ±0.0001 or ±2% REGRESSION
|
|
92
|
+
Total energy 12.48331 12.48329 -0.00002 (-0.00016%) ±0.001% WITHIN TOL
|
|
93
|
+
reconstruction_error: Relative reconstruction error increased by 0.0018 (+24.66%) (0.0073 → 0.0091),
|
|
94
|
+
exceeding the configured tolerance (absolute 0.0001 or relative 2% of |baseline|; allowed change 0.000146).
|
|
95
|
+
The metric is declared 'minimize' (lower is better), so this is a regression.
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Declare the metrics you care about in `.notedelta.yml`:
|
|
99
|
+
|
|
100
|
+
```yaml
|
|
101
|
+
version: 1
|
|
102
|
+
metrics:
|
|
103
|
+
- name: reconstruction_error
|
|
104
|
+
label: Relative reconstruction error
|
|
105
|
+
direction: minimize
|
|
106
|
+
extraction:
|
|
107
|
+
type: regex
|
|
108
|
+
pattern: 'Relative reconstruction error\s*=\s*([0-9.eE+-]+)'
|
|
109
|
+
tolerance: {absolute: 0.0001, relative: 0.02}
|
|
110
|
+
- name: validation_accuracy
|
|
111
|
+
direction: maximize
|
|
112
|
+
unit: fraction # shown as %, differences in percentage points
|
|
113
|
+
extraction: {type: key_value, key: Validation accuracy}
|
|
114
|
+
tolerance: {absolute: 0.002}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Use it as a check, locally or in CI:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
notedelta compare a.ipynb b.ipynb --html report.html --fail-on-regression
|
|
121
|
+
notedelta compare-git notebooks/analysis.ipynb --ref main
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Exit codes: `0` ok, `1` a fail-on condition was met, `2` invalid input or configuration,
|
|
125
|
+
`3` internal error.
|
|
126
|
+
|
|
127
|
+
## GitHub Action
|
|
128
|
+
|
|
129
|
+
This repository is also a GitHub Action that checks notebooks in pull requests. It writes
|
|
130
|
+
a job summary, uploads the HTML and JSON reports and needs only read permission. It
|
|
131
|
+
installs NoteDelta from PyPI.
|
|
132
|
+
|
|
133
|
+
```yaml
|
|
134
|
+
permissions:
|
|
135
|
+
contents: read
|
|
136
|
+
jobs:
|
|
137
|
+
notedelta:
|
|
138
|
+
runs-on: ubuntu-latest
|
|
139
|
+
steps:
|
|
140
|
+
- uses: actions/checkout@v4
|
|
141
|
+
with:
|
|
142
|
+
fetch-depth: 0
|
|
143
|
+
- uses: ouyang-matters/notedelta@v0.1.0
|
|
144
|
+
with:
|
|
145
|
+
notebooks: |
|
|
146
|
+
notebooks/**/*.ipynb
|
|
147
|
+
fail-on: regression,new_error
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
See [docs/ci.md](https://github.com/ouyang-matters/notedelta/blob/main/docs/ci.md) for all inputs.
|
|
151
|
+
|
|
152
|
+
## Documentation
|
|
153
|
+
|
|
154
|
+
| | |
|
|
155
|
+
|---|---|
|
|
156
|
+
| [Getting started](https://github.com/ouyang-matters/notedelta/blob/main/docs/getting-started.md) | a ten-minute tutorial with the bundled [examples](https://github.com/ouyang-matters/notedelta/tree/main/examples) |
|
|
157
|
+
| [CLI reference](https://github.com/ouyang-matters/notedelta/blob/main/docs/cli.md) | every command, option and exit code |
|
|
158
|
+
| [Configuration](https://github.com/ouyang-matters/notedelta/blob/main/docs/configuration.md) | the `.notedelta.yml` v1 specification |
|
|
159
|
+
| [Numerical comparison](https://github.com/ouyang-matters/notedelta/blob/main/docs/numerics.md) | formulas, tolerances, NaN/zero/precision handling, arrays and tables |
|
|
160
|
+
| [Classification rules](https://github.com/ouyang-matters/notedelta/blob/main/docs/classification.md) | every rule that can label a result |
|
|
161
|
+
| [Environment diagnostics](https://github.com/ouyang-matters/notedelta/blob/main/docs/environment.md) | what is inspected and how findings are graded |
|
|
162
|
+
| [Report format](https://github.com/ouyang-matters/notedelta/blob/main/docs/report-format.md) | the versioned JSON report ([schema](https://github.com/ouyang-matters/notedelta/blob/main/docs/report-schema.json)) |
|
|
163
|
+
| [VS Code extension](https://github.com/ouyang-matters/notedelta/blob/main/docs/vscode.md) · [GitHub Actions](https://github.com/ouyang-matters/notedelta/blob/main/docs/ci.md) | integrations |
|
|
164
|
+
| [Security and privacy](https://github.com/ouyang-matters/notedelta/blob/main/docs/security.md) · [Known limitations](https://github.com/ouyang-matters/notedelta/blob/main/docs/limitations.md) | |
|
|
165
|
+
|
|
166
|
+
Example reports generated by NoteDelta: [ML experiment](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/ml-experiment.html),
|
|
167
|
+
[numerical simulation](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/numerical-simulation.html),
|
|
168
|
+
[reproducibility](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/reproducibility.html). Download them and open locally;
|
|
169
|
+
they are self-contained HTML files.
|
|
170
|
+
|
|
171
|
+
## Feedback
|
|
172
|
+
|
|
173
|
+
Bug reports, questions and feature requests are welcome in
|
|
174
|
+
[Issues](https://github.com/ouyang-matters/notedelta/issues). Please report security
|
|
175
|
+
problems privately (see [SECURITY.md](https://github.com/ouyang-matters/notedelta/blob/main/SECURITY.md)).
|
|
176
|
+
|
|
177
|
+
## What NoteDelta is not
|
|
178
|
+
|
|
179
|
+
NoteDelta reports observable differences and applies the rules you configure. It does not
|
|
180
|
+
decide whether an experiment is scientifically valid or reproducible, does not infer
|
|
181
|
+
statistical significance from two values, and does not interpret images (plots are
|
|
182
|
+
compared by content hash). Interpretation remains the researcher's responsibility.
|
|
183
|
+
|
|
184
|
+
## License
|
|
185
|
+
|
|
186
|
+
Proprietary freeware, see [LICENSE](https://github.com/ouyang-matters/notedelta/blob/main/LICENSE). It is free to use, including commercially.
|
|
187
|
+
Third-party components keep their own licences ([THIRD_PARTY_NOTICES.md](https://github.com/ouyang-matters/notedelta/blob/main/THIRD_PARTY_NOTICES.md)).
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# NoteDelta
|
|
2
|
+
|
|
3
|
+
**Know what changed in your scientific results, not just your code.**
|
|
4
|
+
|
|
5
|
+
NoteDelta compares two versions of a Jupyter notebook and answers the question a code
|
|
6
|
+
diff cannot: *which numerical results changed, by how much, does that exceed what I
|
|
7
|
+
consider acceptable, and is there something about the environment I should check?*
|
|
8
|
+
|
|
9
|
+
- **Configured scientific metrics** - declare a metric once (how to find it, whether
|
|
10
|
+
lower or higher is better, a tolerance). NoteDelta reports the exact difference,
|
|
11
|
+
relative change and whether it is a regression, an improvement or within tolerance,
|
|
12
|
+
and explains the rule it applied.
|
|
13
|
+
- **Automatic numerical discovery** - numbers in changed outputs (logs, printed values,
|
|
14
|
+
NumPy arrays, pandas tables, JSON) are paired and compared even without configuration.
|
|
15
|
+
They are reported as changes, never as regressions, because their direction is unknown.
|
|
16
|
+
- **Reproducibility diagnostics** - kernel, Python version, locked package versions,
|
|
17
|
+
dependency declarations, random seeds and execution order, each labelled as a
|
|
18
|
+
*fact*, a *warning* or a *hypothesis*. NoteDelta never claims a cause it cannot show.
|
|
19
|
+
|
|
20
|
+
It runs entirely offline, never executes notebook code during comparison, sends no
|
|
21
|
+
telemetry and needs no account.
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
NoteDelta needs Python 3.11 or newer.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install notedelta
|
|
31
|
+
pip install "notedelta[execute]" # optional: opt-in notebook re-execution
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick start
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
notedelta compare baseline.ipynb experiment.ipynb
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
NoteDelta baseline.ipynb → candidate.ipynb
|
|
42
|
+
1 regression, 2 numerical changes
|
|
43
|
+
cells: +0 −0 ~6 moved 0 | outputs changed 5 | numbers compared 14 | reproducibility warnings 1
|
|
44
|
+
|
|
45
|
+
Scientific metrics
|
|
46
|
+
Metric Baseline Candidate Difference Tolerance Status
|
|
47
|
+
Relative reconstruction error 0.0073 0.0091 +0.0018 (+24.66%) ±0.0001 or ±2% REGRESSION
|
|
48
|
+
Total energy 12.48331 12.48329 -0.00002 (-0.00016%) ±0.001% WITHIN TOL
|
|
49
|
+
Energy drift 1.20e-06 1.21e-06 +1e-8 (+0.83%) ±5% WITHIN TOL
|
|
50
|
+
reconstruction_error: Relative reconstruction error increased by 0.0018 (+24.66%) (0.0073 → 0.0091),
|
|
51
|
+
exceeding the configured tolerance (absolute 0.0001 or relative 2% of |baseline|; allowed change 0.000146).
|
|
52
|
+
The metric is declared 'minimize' (lower is better), so this is a regression.
|
|
53
|
+
|
|
54
|
+
Discovered numerical changes (no declared direction)
|
|
55
|
+
Cell Value Baseline Candidate Difference Status
|
|
56
|
+
#3 Max residual 3.2e-08 4.1e-08 +9e-9 (+28.12%) CHANGED
|
|
57
|
+
#6 Out[6]: u[N // 2, ::16] array (8,) array (8,) max |Δ| 4.843e-05; 7/8 changed CHANGED
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
(Output of `examples/numerical-simulation`, unedited.)
|
|
61
|
+
|
|
62
|
+
Declare the metrics you care about in `.notedelta.yml` (run `notedelta init` for a
|
|
63
|
+
starter file):
|
|
64
|
+
|
|
65
|
+
```yaml
|
|
66
|
+
version: 1
|
|
67
|
+
metrics:
|
|
68
|
+
- name: reconstruction_error
|
|
69
|
+
label: Relative reconstruction error
|
|
70
|
+
direction: minimize
|
|
71
|
+
extraction:
|
|
72
|
+
type: regex
|
|
73
|
+
pattern: 'Relative reconstruction error\s*=\s*([0-9.eE+-]+)'
|
|
74
|
+
tolerance:
|
|
75
|
+
absolute: 0.0001
|
|
76
|
+
relative: 0.02
|
|
77
|
+
- name: validation_accuracy
|
|
78
|
+
direction: maximize
|
|
79
|
+
unit: fraction # shown as %, differences in percentage points
|
|
80
|
+
extraction: {type: key_value, key: Validation accuracy}
|
|
81
|
+
tolerance: {absolute: 0.002}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Reports and CI:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
notedelta compare a.ipynb b.ipynb --html report.html --json report.json
|
|
88
|
+
notedelta compare a.ipynb b.ipynb --fail-on-regression # exit 1 on a configured regression
|
|
89
|
+
notedelta compare-git notebooks/analysis.ipynb --ref main # baseline = the file at a git revision
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
| Exit code | Meaning |
|
|
93
|
+
|---|---|
|
|
94
|
+
| 0 | comparison completed, no fail-on condition met |
|
|
95
|
+
| 1 | a fail-on condition was met (for example `--fail-on-regression`) |
|
|
96
|
+
| 2 | invalid input: unreadable/corrupt notebook, invalid configuration, bad arguments |
|
|
97
|
+
| 3 | internal, git or execution error |
|
|
98
|
+
|
|
99
|
+
## Integrations
|
|
100
|
+
|
|
101
|
+
- **VS Code** (`extensions/vscode`): compare with a git baseline or another notebook,
|
|
102
|
+
results view, HTML report panel, jump to the affected cell. See [docs/vscode.md](docs/vscode.md).
|
|
103
|
+
- **GitHub Actions** (`integrations/github-actions`): check notebooks in pull requests,
|
|
104
|
+
job summary, HTML reports as artifacts, read-only permissions. See [docs/ci.md](docs/ci.md).
|
|
105
|
+
|
|
106
|
+
## Documentation
|
|
107
|
+
|
|
108
|
+
| | |
|
|
109
|
+
|---|---|
|
|
110
|
+
| [Getting started](docs/getting-started.md) | a ten-minute tutorial with the bundled examples |
|
|
111
|
+
| [CLI reference](docs/cli.md) | every command, option and exit code |
|
|
112
|
+
| [Configuration](docs/configuration.md) | the `.notedelta.yml` v1 specification |
|
|
113
|
+
| [Numerical comparison](docs/numerics.md) | formulas, tolerances, NaN/zero/precision handling, arrays and tables |
|
|
114
|
+
| [Classification rules](docs/classification.md) | every rule that can label a result, with its explanation |
|
|
115
|
+
| [Environment diagnostics](docs/environment.md) | what is inspected and how findings are graded |
|
|
116
|
+
| [Report format](docs/report-format.md) | the versioned JSON report |
|
|
117
|
+
| [Architecture](docs/architecture.md) | how the pieces fit together |
|
|
118
|
+
| [Security and privacy](docs/security.md) | threat model, execution, redaction |
|
|
119
|
+
| [Known limitations](docs/limitations.md) | what NoteDelta does not do |
|
|
120
|
+
| [Licensing and editions](docs/licensing.md) | licence, dependencies, provisional Free/Pro boundary |
|
|
121
|
+
| [Contributing](CONTRIBUTING.md) · [Changelog](CHANGELOG.md) · [Release process](docs/release.md) | |
|
|
122
|
+
|
|
123
|
+
## What NoteDelta is not
|
|
124
|
+
|
|
125
|
+
NoteDelta reports observable differences and applies the rules you configure. It does
|
|
126
|
+
not decide whether an experiment is scientifically valid or reproducible, does not
|
|
127
|
+
infer statistical significance from two values, and does not interpret images (plots
|
|
128
|
+
are compared by content hash). Interpretation remains the researcher's responsibility.
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
python -m venv .venv && .venv/Scripts/activate # or: source .venv/bin/activate
|
|
134
|
+
pip install -e ".[dev,execute]"
|
|
135
|
+
pytest # unit, integration, regression tests
|
|
136
|
+
ruff check src tests && ruff format --check src tests && mypy
|
|
137
|
+
python scripts/benchmark.py # performance on a large generated notebook pair
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## License
|
|
141
|
+
|
|
142
|
+
Proprietary freeware, see [LICENSE](LICENSE). Free to use, including commercially.
|
|
143
|
+
Third-party components keep their own licences ([THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)).
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
NoteDelta does not bundle third-party code. It depends on the following packages, which
|
|
4
|
+
pip installs from PyPI under their own licences (versions measured for NoteDelta 0.1.0):
|
|
5
|
+
|
|
6
|
+
| Package | Licence |
|
|
7
|
+
|---|---|
|
|
8
|
+
| nbformat | BSD-3-Clause |
|
|
9
|
+
| pydantic, pydantic-core, annotated-types, typing-inspection | MIT |
|
|
10
|
+
| PyYAML | MIT |
|
|
11
|
+
| packaging | Apache-2.0 OR BSD-2-Clause |
|
|
12
|
+
| jsonschema, jsonschema-specifications, referencing, rpds-py, attrs | MIT |
|
|
13
|
+
| fastjsonschema | BSD-3-Clause |
|
|
14
|
+
| traitlets, jupyter_core | BSD-3-Clause |
|
|
15
|
+
| platformdirs | MIT |
|
|
16
|
+
| typing_extensions | PSF-2.0 |
|
|
17
|
+
|
|
18
|
+
Optional `notedelta[execute]`: nbclient, ipykernel, jupyter_client, pyzmq, IPython,
|
|
19
|
+
psutil (BSD); tornado, asttokens (Apache-2.0); debugpy, jedi, parso (MIT); libzmq
|
|
20
|
+
(bundled in pyzmq wheels, MPL-2.0).
|
|
21
|
+
|
|
22
|
+
The VS Code extension contains only NoteDelta code. Its build tools (TypeScript,
|
|
23
|
+
@vscode/vsce, @vscode/test-electron, @types/*) are development dependencies and are not
|
|
24
|
+
included in the `.vsix`.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
NoteDelta End User License Agreement
|
|
2
|
+
Version 1.0 (draft for the 0.1 beta)
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2026 Ouyang Matters. All rights reserved.
|
|
5
|
+
|
|
6
|
+
This agreement covers the NoteDelta command-line tool and Python package, the NoteDelta
|
|
7
|
+
extension for Visual Studio Code, the NoteDelta GitHub Actions integration and their
|
|
8
|
+
documentation (together, the "Software").
|
|
9
|
+
|
|
10
|
+
1. Licence grant. You may install and use the Software, free of charge, on any number
|
|
11
|
+
of computers, for personal, academic, research and commercial purposes, including
|
|
12
|
+
in continuous-integration systems that you or your organisation operate.
|
|
13
|
+
|
|
14
|
+
2. Restrictions. You may not (a) sell, rent or sublicense the Software, (b) distribute
|
|
15
|
+
modified copies of the Software, or (c) remove or alter copyright or licence notices.
|
|
16
|
+
You may redistribute unmodified copies of the Software, including this agreement,
|
|
17
|
+
for example as part of an internal mirror or a container image.
|
|
18
|
+
|
|
19
|
+
3. Your data. The Software processes your notebooks and project files locally. It
|
|
20
|
+
does not collect telemetry and does not transmit your notebooks, results or source
|
|
21
|
+
code anywhere. Reports it writes stay where you put them.
|
|
22
|
+
|
|
23
|
+
4. Future editions. Some future features may be offered under a separate paid
|
|
24
|
+
licence. Features available free of charge in a released version will not be
|
|
25
|
+
removed from that version, and the Software contains no hidden time limits or
|
|
26
|
+
disabling mechanisms.
|
|
27
|
+
|
|
28
|
+
5. Third-party components. The Software depends on open-source packages that are
|
|
29
|
+
licensed under their own terms (see THIRD_PARTY_NOTICES.md). Nothing in this
|
|
30
|
+
agreement limits your rights under those licences.
|
|
31
|
+
|
|
32
|
+
6. No warranty. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
33
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
34
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. NoteDelta
|
|
35
|
+
reports observable differences between notebooks according to documented rules;
|
|
36
|
+
it does not determine whether results are scientifically valid or reproducible.
|
|
37
|
+
|
|
38
|
+
7. Limitation of liability. TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL
|
|
39
|
+
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
40
|
+
LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR ITS USE.
|
|
41
|
+
|
|
42
|
+
8. Termination. This licence ends automatically if you breach it. Sections 6 and 7
|
|
43
|
+
survive termination.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# NoteDelta for VS Code
|
|
2
|
+
|
|
3
|
+
**Know what changed in your scientific results, not just your code.**
|
|
4
|
+
|
|
5
|
+
NoteDelta compares two versions of a Jupyter notebook and tells you which numerical
|
|
6
|
+
results changed, by how much, whether a configured metric regressed beyond its
|
|
7
|
+
tolerance, and which environment differences (kernel, Python, locked package
|
|
8
|
+
versions, random seeds) deserve a look. It works offline and never runs notebook code.
|
|
9
|
+
|
|
10
|
+
## Commands
|
|
11
|
+
|
|
12
|
+
| Command | What it does |
|
|
13
|
+
|---|---|
|
|
14
|
+
| **NoteDelta: Compare Notebook with Git Baseline** | Compares the saved notebook with the same file at a git revision (default `HEAD`). Also in the notebook toolbar and the Explorer context menu. |
|
|
15
|
+
| **NoteDelta: Compare Notebook with…** | Compares the current notebook with any other `.ipynb` file. |
|
|
16
|
+
| **NoteDelta: Open NoteDelta Report** | Opens the self-contained HTML report beside the notebook (no scripts). |
|
|
17
|
+
| **NoteDelta: Show Numerical Regressions** | Quick pick of regressions and changed numbers; choosing one jumps to the cell. |
|
|
18
|
+
| **NoteDelta: Show Reproducibility Warnings** | Environment facts, warnings and (clearly labelled) hypotheses. |
|
|
19
|
+
| **NoteDelta: Configure Metrics and Tolerances** | Opens `.notedelta.yml`, or creates a starter file after asking. |
|
|
20
|
+
|
|
21
|
+
The **NoteDelta** view in the activity bar lists metrics, numerical changes, environment
|
|
22
|
+
findings and changed cells; clicking an entry opens the corresponding cell.
|
|
23
|
+
|
|
24
|
+
## Requirements
|
|
25
|
+
|
|
26
|
+
The extension is a thin client for the NoteDelta command-line tool (Python 3.11+):
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install notedelta
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
If `notedelta` is not on your `PATH`, set **`notedelta.pythonPath`** to the interpreter
|
|
33
|
+
where it is installed. In untrusted workspaces, workspace-level values of
|
|
34
|
+
`notedelta.pythonPath` and `notedelta.cliPath` are ignored so a repository cannot choose
|
|
35
|
+
which program runs.
|
|
36
|
+
|
|
37
|
+
## Settings
|
|
38
|
+
|
|
39
|
+
- `notedelta.pythonPath`, `notedelta.cliPath` - where to find the CLI.
|
|
40
|
+
- `notedelta.gitRef` - default baseline revision (`HEAD`).
|
|
41
|
+
- `notedelta.configFile` - explicit `.notedelta.yml` (otherwise searched upwards from the notebook).
|
|
42
|
+
- `notedelta.embedImages` - show changed plots side by side in the report.
|
|
43
|
+
- `notedelta.openReportAfterCompare`, `notedelta.timeoutSeconds`.
|
|
44
|
+
|
|
45
|
+
## Privacy
|
|
46
|
+
|
|
47
|
+
No telemetry. Notebooks are read locally by the CLI; nothing is uploaded. Reports are
|
|
48
|
+
written to the extension's storage folder and pruned after 7 days.
|
|
49
|
+
|
|
50
|
+
NoteDelta reports observable differences and applies the rules you configure. Scientific
|
|
51
|
+
interpretation remains yours.
|