notedelta 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. notedelta-0.1.0/.gitignore +33 -0
  2. notedelta-0.1.0/CHANGELOG.md +39 -0
  3. notedelta-0.1.0/LICENSE +43 -0
  4. notedelta-0.1.0/PKG-INFO +187 -0
  5. notedelta-0.1.0/README.md +143 -0
  6. notedelta-0.1.0/THIRD_PARTY_NOTICES.md +24 -0
  7. notedelta-0.1.0/extensions/vscode/CHANGELOG.md +7 -0
  8. notedelta-0.1.0/extensions/vscode/LICENSE +43 -0
  9. notedelta-0.1.0/extensions/vscode/README.md +51 -0
  10. notedelta-0.1.0/public/README.md +146 -0
  11. notedelta-0.1.0/pyproject.toml +105 -0
  12. notedelta-0.1.0/src/notedelta/__init__.py +15 -0
  13. notedelta-0.1.0/src/notedelta/__main__.py +3 -0
  14. notedelta-0.1.0/src/notedelta/_version.py +1 -0
  15. notedelta-0.1.0/src/notedelta/cli.py +473 -0
  16. notedelta-0.1.0/src/notedelta/config/__init__.py +4 -0
  17. notedelta-0.1.0/src/notedelta/config/loader.py +137 -0
  18. notedelta-0.1.0/src/notedelta/config/schema.py +326 -0
  19. notedelta-0.1.0/src/notedelta/engine.py +761 -0
  20. notedelta-0.1.0/src/notedelta/environment/__init__.py +0 -0
  21. notedelta-0.1.0/src/notedelta/environment/deps.py +503 -0
  22. notedelta-0.1.0/src/notedelta/environment/notebook_env.py +183 -0
  23. notedelta-0.1.0/src/notedelta/environment/seeds.py +175 -0
  24. notedelta-0.1.0/src/notedelta/errors.py +44 -0
  25. notedelta-0.1.0/src/notedelta/execution/__init__.py +0 -0
  26. notedelta-0.1.0/src/notedelta/execution/runner.py +143 -0
  27. notedelta-0.1.0/src/notedelta/gitref.py +127 -0
  28. notedelta-0.1.0/src/notedelta/limits.py +29 -0
  29. notedelta-0.1.0/src/notedelta/matching/__init__.py +3 -0
  30. notedelta-0.1.0/src/notedelta/matching/cells.py +284 -0
  31. notedelta-0.1.0/src/notedelta/metrics/__init__.py +0 -0
  32. notedelta-0.1.0/src/notedelta/metrics/extract.py +160 -0
  33. notedelta-0.1.0/src/notedelta/model.py +300 -0
  34. notedelta-0.1.0/src/notedelta/notebooks/__init__.py +4 -0
  35. notedelta-0.1.0/src/notedelta/notebooks/loader.py +259 -0
  36. notedelta-0.1.0/src/notedelta/notebooks/model.py +179 -0
  37. notedelta-0.1.0/src/notedelta/numeric/__init__.py +0 -0
  38. notedelta-0.1.0/src/notedelta/numeric/compare.py +227 -0
  39. notedelta-0.1.0/src/notedelta/numeric/discover.py +181 -0
  40. notedelta-0.1.0/src/notedelta/numeric/literal.py +456 -0
  41. notedelta-0.1.0/src/notedelta/numeric/numbers.py +183 -0
  42. notedelta-0.1.0/src/notedelta/numeric/table.py +264 -0
  43. notedelta-0.1.0/src/notedelta/numeric/text.py +190 -0
  44. notedelta-0.1.0/src/notedelta/regression/__init__.py +0 -0
  45. notedelta-0.1.0/src/notedelta/regression/classify.py +275 -0
  46. notedelta-0.1.0/src/notedelta/reporting/__init__.py +11 -0
  47. notedelta-0.1.0/src/notedelta/reporting/common.py +144 -0
  48. notedelta-0.1.0/src/notedelta/reporting/html.py +503 -0
  49. notedelta-0.1.0/src/notedelta/reporting/markdown.py +104 -0
  50. notedelta-0.1.0/src/notedelta/reporting/redact.py +70 -0
  51. notedelta-0.1.0/src/notedelta/reporting/terminal.py +170 -0
  52. notedelta-0.1.0/src/notedelta/stability.py +111 -0
  53. notedelta-0.1.0/tests/conftest.py +39 -0
  54. notedelta-0.1.0/tests/fixtures/build_fixtures.py +496 -0
  55. notedelta-0.1.0/tests/fixtures/edge/corrupt.ipynb +1 -0
  56. notedelta-0.1.0/tests/fixtures/edge/empty.ipynb +0 -0
  57. notedelta-0.1.0/tests/fixtures/edge/hostile_a.ipynb +85 -0
  58. notedelta-0.1.0/tests/fixtures/edge/hostile_b.ipynb +86 -0
  59. notedelta-0.1.0/tests/fixtures/edge/list.ipynb +1 -0
  60. notedelta-0.1.0/tests/fixtures/edge/no_ids.ipynb +49 -0
  61. notedelta-0.1.0/tests/fixtures/edge/not_a_notebook.ipynb +1 -0
  62. notedelta-0.1.0/tests/fixtures/edge/v3.ipynb +31 -0
  63. notedelta-0.1.0/tests/fixtures/scenario_a_ml/.notedelta.yml +21 -0
  64. notedelta-0.1.0/tests/fixtures/scenario_a_ml/baseline.ipynb +220 -0
  65. notedelta-0.1.0/tests/fixtures/scenario_a_ml/candidate.ipynb +220 -0
  66. notedelta-0.1.0/tests/fixtures/scenario_b_simulation/.notedelta.yml +33 -0
  67. notedelta-0.1.0/tests/fixtures/scenario_b_simulation/baseline.ipynb +150 -0
  68. notedelta-0.1.0/tests/fixtures/scenario_b_simulation/candidate.ipynb +150 -0
  69. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/.notedelta-root +1 -0
  70. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/analysis.ipynb +90 -0
  71. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/requirements.txt +3 -0
  72. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/baseline/uv.lock +14 -0
  73. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/.notedelta-root +1 -0
  74. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/analysis.ipynb +90 -0
  75. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/requirements.txt +3 -0
  76. notedelta-0.1.0/tests/fixtures/scenario_c_reproducibility/candidate/uv.lock +14 -0
  77. notedelta-0.1.0/tests/integration/test_cli.py +147 -0
  78. notedelta-0.1.0/tests/integration/test_execute.py +95 -0
  79. notedelta-0.1.0/tests/integration/test_git.py +104 -0
  80. notedelta-0.1.0/tests/integration/test_github_action.py +167 -0
  81. notedelta-0.1.0/tests/integration/test_scenarios.py +130 -0
  82. notedelta-0.1.0/tests/regression/test_report_schema.py +67 -0
  83. notedelta-0.1.0/tests/unit/test_classify_and_extract.py +192 -0
  84. notedelta-0.1.0/tests/unit/test_compare.py +175 -0
  85. notedelta-0.1.0/tests/unit/test_config.py +137 -0
  86. notedelta-0.1.0/tests/unit/test_environment.py +161 -0
  87. notedelta-0.1.0/tests/unit/test_literal_and_text.py +109 -0
  88. notedelta-0.1.0/tests/unit/test_loader.py +136 -0
  89. notedelta-0.1.0/tests/unit/test_matching.py +160 -0
  90. notedelta-0.1.0/tests/unit/test_numbers.py +98 -0
  91. notedelta-0.1.0/tests/unit/test_report_safety.py +92 -0
@@ -0,0 +1,33 @@
1
+ # Python
2
+ .venv/
3
+ venv/
4
+ __pycache__/
5
+ *.py[cod]
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .mypy_cache/
9
+ .ruff_cache/
10
+ .coverage
11
+ coverage.xml
12
+ htmlcov/
13
+ build/
14
+ dist/
15
+
16
+ # NoteDelta outputs
17
+ notedelta-runs/
18
+ notedelta-reports/
19
+ *.notedelta.html
20
+ /report*.html
21
+ /report*.json
22
+
23
+ # Node / VS Code extension
24
+ node_modules/
25
+ extensions/vscode/out/
26
+ extensions/vscode/*.vsix
27
+ .vscode-test/
28
+
29
+ # Editors / OS
30
+ .idea/
31
+ .DS_Store
32
+ Thumbs.db
33
+ .ipynb_checkpoints/
@@ -0,0 +1,39 @@
1
+ # Changelog
2
+
3
+ All notable changes are listed here. Versions follow semantic versioning; the JSON
4
+ report and the configuration carry their own schema versions.
5
+
6
+ ## 0.1.0 - 2026-11-15 (first beta)
7
+
8
+ - Renamed from the working name LabTrace to NoteDelta before the first release (the
9
+ `labtrace` name was taken on PyPI). Package, command, configuration file
10
+ (`.notedelta.yml`) and VS Code identifiers all use `notedelta`.
11
+
12
+ Core and CLI
13
+ - `notedelta compare`: cell matching by stable ids, identical content and similarity,
14
+ with reorder detection and explicit ambiguity flags; source, output, error, execution
15
+ count and metadata diffs; image outputs compared by hash.
16
+ - Numerical comparison with exact decimal arithmetic, absolute/relative tolerances,
17
+ careful handling of zero baselines, negative values, NaN, infinities and display
18
+ precision; arrays (shape check, elementwise, max/mean/RMS/relative L2) and pandas
19
+ tables.
20
+ - Automatic discovery of numbers in logs, printed values, NumPy arrays, JSON outputs
21
+ and tables, via template-aligned text and a safe literal parser.
22
+ - `.notedelta.yml` v1: configured metrics (regex, key/value, JSON path extraction),
23
+ directions, units, tolerances, cell selectors, discovery and policy settings; strict
24
+ validation and regex safety checks.
25
+ - Explainable classification (regression, improvement, change, within tolerance,
26
+ missing, incomparable).
27
+ - Environment diagnostics: kernel, Python version, interpreter, execution order, seeds,
28
+ requirements/pyproject/uv/poetry/Pipfile/conda files, graded as fact/warning/hypothesis.
29
+ - Reports: terminal, JSON (schema 1.0, JSON Schema via `notedelta schema`), Markdown,
30
+ self-contained script-free HTML with light/dark themes. Best-effort redaction.
31
+ - `notedelta compare-git`, `check-config`, `init`, `schema`.
32
+ - Opt-in `notedelta execute` (nbclient; timeouts, repeats, provenance; not sandboxed) and
33
+ `notedelta stability`.
34
+
35
+ Integrations
36
+ - VS Code extension (preview): compare with git baseline or file, results view,
37
+ report panel, jump to cell, regression/reproducibility quick picks, starter config.
38
+ - GitHub composite action with job summary, report artifact and least-privilege example
39
+ workflow.
@@ -0,0 +1,43 @@
1
+ NoteDelta End User License Agreement
2
+ Version 1.0 (draft for the 0.1 beta)
3
+
4
+ Copyright (c) 2026 Ouyang Matters. All rights reserved.
5
+
6
+ This agreement covers the NoteDelta command-line tool and Python package, the NoteDelta
7
+ extension for Visual Studio Code, the NoteDelta GitHub Actions integration and their
8
+ documentation (together, the "Software").
9
+
10
+ 1. Licence grant. You may install and use the Software, free of charge, on any number
11
+ of computers, for personal, academic, research and commercial purposes, including
12
+ in continuous-integration systems that you or your organisation operate.
13
+
14
+ 2. Restrictions. You may not (a) sell, rent or sublicense the Software, (b) distribute
15
+ modified copies of the Software, or (c) remove or alter copyright or licence notices.
16
+ You may redistribute unmodified copies of the Software, including this agreement,
17
+ for example as part of an internal mirror or a container image.
18
+
19
+ 3. Your data. The Software processes your notebooks and project files locally. It
20
+ does not collect telemetry and does not transmit your notebooks, results or source
21
+ code anywhere. Reports it writes stay where you put them.
22
+
23
+ 4. Future editions. Some future features may be offered under a separate paid
24
+ licence. Features available free of charge in a released version will not be
25
+ removed from that version, and the Software contains no hidden time limits or
26
+ disabling mechanisms.
27
+
28
+ 5. Third-party components. The Software depends on open-source packages that are
29
+ licensed under their own terms (see THIRD_PARTY_NOTICES.md). Nothing in this
30
+ agreement limits your rights under those licences.
31
+
32
+ 6. No warranty. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
33
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
34
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. NoteDelta
35
+ reports observable differences between notebooks according to documented rules;
36
+ it does not determine whether results are scientifically valid or reproducible.
37
+
38
+ 7. Limitation of liability. TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL
39
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
40
+ LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR ITS USE.
41
+
42
+ 8. Termination. This licence ends automatically if you breach it. Sections 6 and 7
43
+ survive termination.
@@ -0,0 +1,187 @@
1
+ Metadata-Version: 2.5
2
+ Name: notedelta
3
+ Version: 0.1.0
4
+ Summary: Scientific notebook review: what changed in your numerical results, how much, and can you trust the comparison.
5
+ Project-URL: Homepage, https://github.com/ouyang-matters/notedelta
6
+ Project-URL: Documentation, https://github.com/ouyang-matters/notedelta/tree/main/docs
7
+ Project-URL: Issues, https://github.com/ouyang-matters/notedelta/issues
8
+ Project-URL: Changelog, https://github.com/ouyang-matters/notedelta/blob/main/CHANGELOG.md
9
+ Author: Ouyang Matters
10
+ License-Expression: LicenseRef-NoteDelta-EULA
11
+ License-File: LICENSE
12
+ Keywords: diff,jupyter,notebook,numerical,regression,reproducibility,scientific computing
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Environment :: Console
15
+ Classifier: Framework :: Jupyter
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering
24
+ Classifier: Topic :: Software Development :: Quality Assurance
25
+ Requires-Python: >=3.11
26
+ Requires-Dist: nbformat>=5.7
27
+ Requires-Dist: packaging>=23.0
28
+ Requires-Dist: pydantic<3,>=2.6
29
+ Requires-Dist: pyyaml>=6.0
30
+ Provides-Extra: dev
31
+ Requires-Dist: build>=1.2; extra == 'dev'
32
+ Requires-Dist: mypy>=1.10; extra == 'dev'
33
+ Requires-Dist: pytest-cov>=5; extra == 'dev'
34
+ Requires-Dist: pytest>=8; extra == 'dev'
35
+ Requires-Dist: ruff>=0.6; extra == 'dev'
36
+ Requires-Dist: types-pyyaml; extra == 'dev'
37
+ Provides-Extra: execute
38
+ Requires-Dist: ipykernel>=6.0; extra == 'execute'
39
+ Requires-Dist: nbclient>=0.9; extra == 'execute'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # NoteDelta
43
+
44
+ **Know what changed in your scientific results, not just your code.**
45
+
46
+ NoteDelta compares two versions of a Jupyter notebook and answers the question a code diff
47
+ cannot: *which numerical results changed, by how much, does that exceed what I consider
48
+ acceptable, and is there something about the environment I should check?*
49
+
50
+ - **Configured scientific metrics.** Declare a metric once: how to find it, whether
51
+ lower or higher is better, and a tolerance. NoteDelta reports the exact difference,
52
+ the relative change, and whether it is a regression, an improvement or within
53
+ tolerance, and explains the rule it applied.
54
+ - **Automatic numerical discovery.** Numbers in changed outputs (logs, printed values,
55
+ NumPy arrays, pandas tables, JSON) are paired and compared without configuration.
56
+ Because their direction is unknown, they are reported as changes, never as regressions.
57
+ - **Reproducibility diagnostics.** Differences in kernel, Python version, locked package
58
+ versions, dependency declarations, random seeds and execution order are each labelled
59
+ as a *fact*, a *warning* or a *hypothesis*. NoteDelta never claims a cause it cannot show.
60
+
61
+ NoteDelta runs entirely offline, never executes notebook code during comparison, sends no
62
+ telemetry and needs no account.
63
+
64
+ ![NoteDelta HTML report](https://raw.githubusercontent.com/ouyang-matters/notedelta/main/docs/images/report-light.png)
65
+
66
+ > **Status: beta (0.1).** The VS Code extension and the GitHub Marketplace listing are in
67
+ > preparation. This repository hosts the documentation, the issue tracker, the licence and
68
+ > the GitHub Action. The source code is not public.
69
+
70
+ ## Install
71
+
72
+ NoteDelta needs Python 3.11 or newer and runs on Windows, macOS and Linux.
73
+
74
+ ```bash
75
+ pip install notedelta
76
+ pip install "notedelta[execute]" # optional: opt-in notebook re-execution (runs code; not sandboxed)
77
+ ```
78
+
79
+ ## Quick look
80
+
81
+ ```bash
82
+ notedelta compare baseline.ipynb experiment.ipynb
83
+ ```
84
+
85
+ ```text
86
+ NoteDelta baseline.ipynb → candidate.ipynb
87
+ 1 regression, 2 numerical changes
88
+
89
+ Scientific metrics
90
+ Metric Baseline Candidate Difference Tolerance Status
91
+ Relative reconstruction error 0.0073 0.0091 +0.0018 (+24.66%) ±0.0001 or ±2% REGRESSION
92
+ Total energy 12.48331 12.48329 -0.00002 (-0.00016%) ±0.001% WITHIN TOL
93
+ reconstruction_error: Relative reconstruction error increased by 0.0018 (+24.66%) (0.0073 → 0.0091),
94
+ exceeding the configured tolerance (absolute 0.0001 or relative 2% of |baseline|; allowed change 0.000146).
95
+ The metric is declared 'minimize' (lower is better), so this is a regression.
96
+ ```
97
+
98
+ Declare the metrics you care about in `.notedelta.yml`:
99
+
100
+ ```yaml
101
+ version: 1
102
+ metrics:
103
+ - name: reconstruction_error
104
+ label: Relative reconstruction error
105
+ direction: minimize
106
+ extraction:
107
+ type: regex
108
+ pattern: 'Relative reconstruction error\s*=\s*([0-9.eE+-]+)'
109
+ tolerance: {absolute: 0.0001, relative: 0.02}
110
+ - name: validation_accuracy
111
+ direction: maximize
112
+ unit: fraction # shown as %, differences in percentage points
113
+ extraction: {type: key_value, key: Validation accuracy}
114
+ tolerance: {absolute: 0.002}
115
+ ```
116
+
117
+ Use it as a check, locally or in CI:
118
+
119
+ ```bash
120
+ notedelta compare a.ipynb b.ipynb --html report.html --fail-on-regression
121
+ notedelta compare-git notebooks/analysis.ipynb --ref main
122
+ ```
123
+
124
+ Exit codes: `0` ok, `1` a fail-on condition was met, `2` invalid input or configuration,
125
+ `3` internal error.
126
+
127
+ ## GitHub Action
128
+
129
+ This repository is also a GitHub Action that checks notebooks in pull requests. It writes
130
+ a job summary, uploads the HTML and JSON reports and needs only read permission. It
131
+ installs NoteDelta from PyPI.
132
+
133
+ ```yaml
134
+ permissions:
135
+ contents: read
136
+ jobs:
137
+ notedelta:
138
+ runs-on: ubuntu-latest
139
+ steps:
140
+ - uses: actions/checkout@v4
141
+ with:
142
+ fetch-depth: 0
143
+ - uses: ouyang-matters/notedelta@v0.1.0
144
+ with:
145
+ notebooks: |
146
+ notebooks/**/*.ipynb
147
+ fail-on: regression,new_error
148
+ ```
149
+
150
+ See [docs/ci.md](https://github.com/ouyang-matters/notedelta/blob/main/docs/ci.md) for all inputs.
151
+
152
+ ## Documentation
153
+
154
+ | | |
155
+ |---|---|
156
+ | [Getting started](https://github.com/ouyang-matters/notedelta/blob/main/docs/getting-started.md) | a ten-minute tutorial with the bundled [examples](https://github.com/ouyang-matters/notedelta/tree/main/examples) |
157
+ | [CLI reference](https://github.com/ouyang-matters/notedelta/blob/main/docs/cli.md) | every command, option and exit code |
158
+ | [Configuration](https://github.com/ouyang-matters/notedelta/blob/main/docs/configuration.md) | the `.notedelta.yml` v1 specification |
159
+ | [Numerical comparison](https://github.com/ouyang-matters/notedelta/blob/main/docs/numerics.md) | formulas, tolerances, NaN/zero/precision handling, arrays and tables |
160
+ | [Classification rules](https://github.com/ouyang-matters/notedelta/blob/main/docs/classification.md) | every rule that can label a result |
161
+ | [Environment diagnostics](https://github.com/ouyang-matters/notedelta/blob/main/docs/environment.md) | what is inspected and how findings are graded |
162
+ | [Report format](https://github.com/ouyang-matters/notedelta/blob/main/docs/report-format.md) | the versioned JSON report ([schema](https://github.com/ouyang-matters/notedelta/blob/main/docs/report-schema.json)) |
163
+ | [VS Code extension](https://github.com/ouyang-matters/notedelta/blob/main/docs/vscode.md) · [GitHub Actions](https://github.com/ouyang-matters/notedelta/blob/main/docs/ci.md) | integrations |
164
+ | [Security and privacy](https://github.com/ouyang-matters/notedelta/blob/main/docs/security.md) · [Known limitations](https://github.com/ouyang-matters/notedelta/blob/main/docs/limitations.md) | |
165
+
166
+ Example reports generated by NoteDelta: [ML experiment](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/ml-experiment.html),
167
+ [numerical simulation](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/numerical-simulation.html),
168
+ [reproducibility](https://github.com/ouyang-matters/notedelta/blob/main/examples/reports/reproducibility.html). Download them and open locally;
169
+ they are self-contained HTML files.
170
+
171
+ ## Feedback
172
+
173
+ Bug reports, questions and feature requests are welcome in
174
+ [Issues](https://github.com/ouyang-matters/notedelta/issues). Please report security
175
+ problems privately (see [SECURITY.md](https://github.com/ouyang-matters/notedelta/blob/main/SECURITY.md)).
176
+
177
+ ## What NoteDelta is not
178
+
179
+ NoteDelta reports observable differences and applies the rules you configure. It does not
180
+ decide whether an experiment is scientifically valid or reproducible, does not infer
181
+ statistical significance from two values, and does not interpret images (plots are
182
+ compared by content hash). Interpretation remains the researcher's responsibility.
183
+
184
+ ## License
185
+
186
+ Proprietary freeware, see [LICENSE](https://github.com/ouyang-matters/notedelta/blob/main/LICENSE). It is free to use, including commercially.
187
+ Third-party components keep their own licences ([THIRD_PARTY_NOTICES.md](https://github.com/ouyang-matters/notedelta/blob/main/THIRD_PARTY_NOTICES.md)).
@@ -0,0 +1,143 @@
1
+ # NoteDelta
2
+
3
+ **Know what changed in your scientific results, not just your code.**
4
+
5
+ NoteDelta compares two versions of a Jupyter notebook and answers the question a code
6
+ diff cannot: *which numerical results changed, by how much, does that exceed what I
7
+ consider acceptable, and is there something about the environment I should check?*
8
+
9
+ - **Configured scientific metrics** - declare a metric once (how to find it, whether
10
+ lower or higher is better, a tolerance). NoteDelta reports the exact difference,
11
+ relative change and whether it is a regression, an improvement or within tolerance,
12
+ and explains the rule it applied.
13
+ - **Automatic numerical discovery** - numbers in changed outputs (logs, printed values,
14
+ NumPy arrays, pandas tables, JSON) are paired and compared even without configuration.
15
+ They are reported as changes, never as regressions, because their direction is unknown.
16
+ - **Reproducibility diagnostics** - kernel, Python version, locked package versions,
17
+ dependency declarations, random seeds and execution order, each labelled as a
18
+ *fact*, a *warning* or a *hypothesis*. NoteDelta never claims a cause it cannot show.
19
+
20
+ It runs entirely offline, never executes notebook code during comparison, sends no
21
+ telemetry and needs no account.
22
+
23
+ ![NoteDelta HTML report](docs/images/report-light.png)
24
+
25
+ ## Install
26
+
27
+ NoteDelta needs Python 3.11 or newer.
28
+
29
+ ```bash
30
+ pip install notedelta
31
+ pip install "notedelta[execute]" # optional: opt-in notebook re-execution
32
+ ```
33
+
34
+ ## Quick start
35
+
36
+ ```bash
37
+ notedelta compare baseline.ipynb experiment.ipynb
38
+ ```
39
+
40
+ ```text
41
+ NoteDelta baseline.ipynb → candidate.ipynb
42
+ 1 regression, 2 numerical changes
43
+ cells: +0 −0 ~6 moved 0 | outputs changed 5 | numbers compared 14 | reproducibility warnings 1
44
+
45
+ Scientific metrics
46
+ Metric Baseline Candidate Difference Tolerance Status
47
+ Relative reconstruction error 0.0073 0.0091 +0.0018 (+24.66%) ±0.0001 or ±2% REGRESSION
48
+ Total energy 12.48331 12.48329 -0.00002 (-0.00016%) ±0.001% WITHIN TOL
49
+ Energy drift 1.20e-06 1.21e-06 +1e-8 (+0.83%) ±5% WITHIN TOL
50
+ reconstruction_error: Relative reconstruction error increased by 0.0018 (+24.66%) (0.0073 → 0.0091),
51
+ exceeding the configured tolerance (absolute 0.0001 or relative 2% of |baseline|; allowed change 0.000146).
52
+ The metric is declared 'minimize' (lower is better), so this is a regression.
53
+
54
+ Discovered numerical changes (no declared direction)
55
+ Cell Value Baseline Candidate Difference Status
56
+ #3 Max residual 3.2e-08 4.1e-08 +9e-9 (+28.12%) CHANGED
57
+ #6 Out[6]: u[N // 2, ::16] array (8,) array (8,) max |Δ| 4.843e-05; 7/8 changed CHANGED
58
+ ```
59
+
60
+ (Output of `examples/numerical-simulation`, unedited.)
61
+
62
+ Declare the metrics you care about in `.notedelta.yml` (run `notedelta init` for a
63
+ starter file):
64
+
65
+ ```yaml
66
+ version: 1
67
+ metrics:
68
+ - name: reconstruction_error
69
+ label: Relative reconstruction error
70
+ direction: minimize
71
+ extraction:
72
+ type: regex
73
+ pattern: 'Relative reconstruction error\s*=\s*([0-9.eE+-]+)'
74
+ tolerance:
75
+ absolute: 0.0001
76
+ relative: 0.02
77
+ - name: validation_accuracy
78
+ direction: maximize
79
+ unit: fraction # shown as %, differences in percentage points
80
+ extraction: {type: key_value, key: Validation accuracy}
81
+ tolerance: {absolute: 0.002}
82
+ ```
83
+
84
+ Reports and CI:
85
+
86
+ ```bash
87
+ notedelta compare a.ipynb b.ipynb --html report.html --json report.json
88
+ notedelta compare a.ipynb b.ipynb --fail-on-regression # exit 1 on a configured regression
89
+ notedelta compare-git notebooks/analysis.ipynb --ref main # baseline = the file at a git revision
90
+ ```
91
+
92
+ | Exit code | Meaning |
93
+ |---|---|
94
+ | 0 | comparison completed, no fail-on condition met |
95
+ | 1 | a fail-on condition was met (for example `--fail-on-regression`) |
96
+ | 2 | invalid input: unreadable/corrupt notebook, invalid configuration, bad arguments |
97
+ | 3 | internal, git or execution error |
98
+
99
+ ## Integrations
100
+
101
+ - **VS Code** (`extensions/vscode`): compare with a git baseline or another notebook,
102
+ results view, HTML report panel, jump to the affected cell. See [docs/vscode.md](docs/vscode.md).
103
+ - **GitHub Actions** (`integrations/github-actions`): check notebooks in pull requests,
104
+ job summary, HTML reports as artifacts, read-only permissions. See [docs/ci.md](docs/ci.md).
105
+
106
+ ## Documentation
107
+
108
+ | | |
109
+ |---|---|
110
+ | [Getting started](docs/getting-started.md) | a ten-minute tutorial with the bundled examples |
111
+ | [CLI reference](docs/cli.md) | every command, option and exit code |
112
+ | [Configuration](docs/configuration.md) | the `.notedelta.yml` v1 specification |
113
+ | [Numerical comparison](docs/numerics.md) | formulas, tolerances, NaN/zero/precision handling, arrays and tables |
114
+ | [Classification rules](docs/classification.md) | every rule that can label a result, with its explanation |
115
+ | [Environment diagnostics](docs/environment.md) | what is inspected and how findings are graded |
116
+ | [Report format](docs/report-format.md) | the versioned JSON report |
117
+ | [Architecture](docs/architecture.md) | how the pieces fit together |
118
+ | [Security and privacy](docs/security.md) | threat model, execution, redaction |
119
+ | [Known limitations](docs/limitations.md) | what NoteDelta does not do |
120
+ | [Licensing and editions](docs/licensing.md) | licence, dependencies, provisional Free/Pro boundary |
121
+ | [Contributing](CONTRIBUTING.md) · [Changelog](CHANGELOG.md) · [Release process](docs/release.md) | |
122
+
123
+ ## What NoteDelta is not
124
+
125
+ NoteDelta reports observable differences and applies the rules you configure. It does
126
+ not decide whether an experiment is scientifically valid or reproducible, does not
127
+ infer statistical significance from two values, and does not interpret images (plots
128
+ are compared by content hash). Interpretation remains the researcher's responsibility.
129
+
130
+ ## Development
131
+
132
+ ```bash
133
+ python -m venv .venv && .venv/Scripts/activate # or: source .venv/bin/activate
134
+ pip install -e ".[dev,execute]"
135
+ pytest # unit, integration, regression tests
136
+ ruff check src tests && ruff format --check src tests && mypy
137
+ python scripts/benchmark.py # performance on a large generated notebook pair
138
+ ```
139
+
140
+ ## License
141
+
142
+ Proprietary freeware, see [LICENSE](LICENSE). Free to use, including commercially.
143
+ Third-party components keep their own licences ([THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)).
@@ -0,0 +1,24 @@
1
+ # Third-party notices
2
+
3
+ NoteDelta does not bundle third-party code. It depends on the following packages, which
4
+ pip installs from PyPI under their own licences (versions measured for NoteDelta 0.1.0):
5
+
6
+ | Package | Licence |
7
+ |---|---|
8
+ | nbformat | BSD-3-Clause |
9
+ | pydantic, pydantic-core, annotated-types, typing-inspection | MIT |
10
+ | PyYAML | MIT |
11
+ | packaging | Apache-2.0 OR BSD-2-Clause |
12
+ | jsonschema, jsonschema-specifications, referencing, rpds-py, attrs | MIT |
13
+ | fastjsonschema | BSD-3-Clause |
14
+ | traitlets, jupyter_core | BSD-3-Clause |
15
+ | platformdirs | MIT |
16
+ | typing_extensions | PSF-2.0 |
17
+
18
+ Optional `notedelta[execute]`: nbclient, ipykernel, jupyter_client, pyzmq, IPython,
19
+ psutil (BSD); tornado, asttokens (Apache-2.0); debugpy, jedi, parso (MIT); libzmq
20
+ (bundled in pyzmq wheels, MPL-2.0).
21
+
22
+ The VS Code extension contains only NoteDelta code. Its build tools (TypeScript,
23
+ @vscode/vsce, @vscode/test-electron, @types/*) are development dependencies and are not
24
+ included in the `.vsix`.
@@ -0,0 +1,7 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 (unreleased)
4
+
5
+ - First preview: compare with another notebook or a git baseline, results view, HTML
6
+ report panel, regression and reproducibility quick picks, jump to cell, starter
7
+ configuration.
@@ -0,0 +1,43 @@
1
+ NoteDelta End User License Agreement
2
+ Version 1.0 (draft for the 0.1 beta)
3
+
4
+ Copyright (c) 2026 Ouyang Matters. All rights reserved.
5
+
6
+ This agreement covers the NoteDelta command-line tool and Python package, the NoteDelta
7
+ extension for Visual Studio Code, the NoteDelta GitHub Actions integration and their
8
+ documentation (together, the "Software").
9
+
10
+ 1. Licence grant. You may install and use the Software, free of charge, on any number
11
+ of computers, for personal, academic, research and commercial purposes, including
12
+ in continuous-integration systems that you or your organisation operate.
13
+
14
+ 2. Restrictions. You may not (a) sell, rent or sublicense the Software, (b) distribute
15
+ modified copies of the Software, or (c) remove or alter copyright or licence notices.
16
+ You may redistribute unmodified copies of the Software, including this agreement,
17
+ for example as part of an internal mirror or a container image.
18
+
19
+ 3. Your data. The Software processes your notebooks and project files locally. It
20
+ does not collect telemetry and does not transmit your notebooks, results or source
21
+ code anywhere. Reports it writes stay where you put them.
22
+
23
+ 4. Future editions. Some future features may be offered under a separate paid
24
+ licence. Features available free of charge in a released version will not be
25
+ removed from that version, and the Software contains no hidden time limits or
26
+ disabling mechanisms.
27
+
28
+ 5. Third-party components. The Software depends on open-source packages that are
29
+ licensed under their own terms (see THIRD_PARTY_NOTICES.md). Nothing in this
30
+ agreement limits your rights under those licences.
31
+
32
+ 6. No warranty. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
33
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
34
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. NoteDelta
35
+ reports observable differences between notebooks according to documented rules;
36
+ it does not determine whether results are scientifically valid or reproducible.
37
+
38
+ 7. Limitation of liability. TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL
39
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
40
+ LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR ITS USE.
41
+
42
+ 8. Termination. This licence ends automatically if you breach it. Sections 6 and 7
43
+ survive termination.
@@ -0,0 +1,51 @@
1
+ # NoteDelta for VS Code
2
+
3
+ **Know what changed in your scientific results, not just your code.**
4
+
5
+ NoteDelta compares two versions of a Jupyter notebook and tells you which numerical
6
+ results changed, by how much, whether a configured metric regressed beyond its
7
+ tolerance, and which environment differences (kernel, Python, locked package
8
+ versions, random seeds) deserve a look. It works offline and never runs notebook code.
9
+
10
+ ## Commands
11
+
12
+ | Command | What it does |
13
+ |---|---|
14
+ | **NoteDelta: Compare Notebook with Git Baseline** | Compares the saved notebook with the same file at a git revision (default `HEAD`). Also in the notebook toolbar and the Explorer context menu. |
15
+ | **NoteDelta: Compare Notebook with…** | Compares the current notebook with any other `.ipynb` file. |
16
+ | **NoteDelta: Open NoteDelta Report** | Opens the self-contained HTML report beside the notebook (no scripts). |
17
+ | **NoteDelta: Show Numerical Regressions** | Quick pick of regressions and changed numbers; choosing one jumps to the cell. |
18
+ | **NoteDelta: Show Reproducibility Warnings** | Environment facts, warnings and (clearly labelled) hypotheses. |
19
+ | **NoteDelta: Configure Metrics and Tolerances** | Opens `.notedelta.yml`, or creates a starter file after asking. |
20
+
21
+ The **NoteDelta** view in the activity bar lists metrics, numerical changes, environment
22
+ findings and changed cells; clicking an entry opens the corresponding cell.
23
+
24
+ ## Requirements
25
+
26
+ The extension is a thin client for the NoteDelta command-line tool (Python 3.11+):
27
+
28
+ ```bash
29
+ pip install notedelta
30
+ ```
31
+
32
+ If `notedelta` is not on your `PATH`, set **`notedelta.pythonPath`** to the interpreter
33
+ where it is installed. In untrusted workspaces, workspace-level values of
34
+ `notedelta.pythonPath` and `notedelta.cliPath` are ignored so a repository cannot choose
35
+ which program runs.
36
+
37
+ ## Settings
38
+
39
+ - `notedelta.pythonPath`, `notedelta.cliPath` - where to find the CLI.
40
+ - `notedelta.gitRef` - default baseline revision (`HEAD`).
41
+ - `notedelta.configFile` - explicit `.notedelta.yml` (otherwise searched upwards from the notebook).
42
+ - `notedelta.embedImages` - show changed plots side by side in the report.
43
+ - `notedelta.openReportAfterCompare`, `notedelta.timeoutSeconds`.
44
+
45
+ ## Privacy
46
+
47
+ No telemetry. Notebooks are read locally by the CLI; nothing is uploaded. Reports are
48
+ written to the extension's storage folder and pruned after 7 days.
49
+
50
+ NoteDelta reports observable differences and applies the rules you configure. Scientific
51
+ interpretation remains yours.