physverify 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {physverify-0.3.0/src/physverify.egg-info → physverify-0.4.1}/PKG-INFO +89 -7
- {physverify-0.3.0 → physverify-0.4.1}/README.md +88 -6
- {physverify-0.3.0 → physverify-0.4.1}/pyproject.toml +1 -1
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/__init__.py +4 -2
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/agent.py +1 -1
- physverify-0.4.1/src/physverify/butcher.py +296 -0
- physverify-0.4.1/src/physverify/checks.py +145 -0
- physverify-0.4.1/src/physverify/cli.py +192 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/loaders.py +17 -3
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/mms.py +3 -3
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/order.py +11 -1
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/pytest_plugin.py +3 -1
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/references.py +115 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/report.py +29 -4
- physverify-0.4.1/src/physverify/scaffold.py +111 -0
- physverify-0.4.1/src/physverify/smoke.py +62 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/matrix.py +1 -1
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/studies.py +113 -5
- {physverify-0.3.0 → physverify-0.4.1/src/physverify.egg-info}/PKG-INFO +89 -7
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify.egg-info/SOURCES.txt +7 -1
- physverify-0.4.1/tests/test_batch7_lessons.py +140 -0
- physverify-0.4.1/tests/test_butcher_properties.py +69 -0
- physverify-0.4.1/tests/test_scaffold.py +146 -0
- physverify-0.4.1/tests/test_v040_helpers.py +168 -0
- physverify-0.3.0/src/physverify/butcher.py +0 -111
- physverify-0.3.0/src/physverify/checks.py +0 -59
- physverify-0.3.0/src/physverify/cli.py +0 -105
- {physverify-0.3.0 → physverify-0.4.1}/LICENSE +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/setup.cfg +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/__main__.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/__init__.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/double_pendulum.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/lorenz.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/projectile_v1.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/projectile_v2.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/adapters/three_body.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/compare.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/constants.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/matrix.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/method_bugs.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/__init__.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/_common.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/heat1d.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/kepler.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/pendulum.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/projectile.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/problems/sod.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/steppers.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/__init__.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/calibrate.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/checks.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/exact.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/ising.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/reference.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/sampler.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/stochastic/stats.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify/symbolic.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify.egg-info/dependency_links.txt +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify.egg-info/entry_points.txt +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify.egg-info/requires.txt +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/src/physverify.egg-info/top_level.txt +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_agent.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_cli.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_corpus_lessons.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_ising.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_method_bugs.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_mms.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_new_problems.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_order.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_properties.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_real_projects.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_reference_problems.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_references.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_seeded_bugs.py +0 -0
- {physverify-0.3.0 → physverify-0.4.1}/tests/test_studies.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: physverify
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Verification checks for physics simulation code: manufactured solutions, order-of-accuracy studies and a seeded-bug benchmark.
|
|
5
5
|
Author: Ram Chandra Giri
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,7 +39,15 @@ Dynamic: license-file
|
|
|
39
39
|
|
|
40
40
|
**A tool that checks whether a physics simulation is solving the right equations correctly, not just running without errors.**
|
|
41
41
|
|
|
42
|
-
Status: early version (0.
|
|
42
|
+
Status: early version (0.4.0). It has been tried on 250 public physics repositories (see below), but nobody besides the author has used it yet, so feedback is very welcome.
|
|
43
|
+
|
|
44
|
+
## Who it is for
|
|
45
|
+
|
|
46
|
+
- Students and researchers who wrote their own solver (orbits, pendulums, heat flow, shock tubes, Monte Carlo) and want more confidence than "the plot looks right".
|
|
47
|
+
- Maintainers of numerical libraries who want their methods checked on every push, for example that each Runge-Kutta table really has the order its name says.
|
|
48
|
+
- Anyone reviewing someone else's simulation code.
|
|
49
|
+
|
|
50
|
+
You do not need to know numerical analysis to read the results: each check says in plain words what it measured and what it expected.
|
|
43
51
|
|
|
44
52
|
## The problem it addresses
|
|
45
53
|
|
|
@@ -56,10 +64,31 @@ physverify uses a few standard ideas from numerical analysis:
|
|
|
56
64
|
5. **Exact "patch" tests.** Some solutions should be reproduced to computer precision. This catches small bugs that the other checks let through.
|
|
57
65
|
6. **Algebraic checks on the method itself.** For a Runge-Kutta method, whether the coefficients really have the order they claim can be settled exactly, with no test problem (the "order conditions").
|
|
58
66
|
|
|
59
|
-
Each check reports **PASS**, **FAIL**, **EXACT** or **INCONCLUSIVE
|
|
67
|
+
Each check reports **PASS**, **FAIL**, **EXACT** or **INCONCLUSIVE** (or **SKIP** when an optional dependency is missing).
|
|
60
68
|
|
|
61
69
|
There is also a separate set of checks for random (Monte Carlo) simulations, using the 2D Ising model as the example.
|
|
62
70
|
|
|
71
|
+
### What a result looks like
|
|
72
|
+
|
|
73
|
+
```text
|
|
74
|
+
mms PASS midpoint damped oscillator: observed order 2.04 (expected 2)
|
|
75
|
+
reference PASS midpoint damped oscillator: observed order 2.01 (expected 2)
|
|
76
|
+
mms PASS midpoint damped oscillator (whole trajectory): observed order 2.02 (expected 2)
|
|
77
|
+
patch EXACT midpoint damped oscillator: reproduced to round-off (largest error 2.7e-15), as designed
|
|
78
|
+
4 of 4 checks passed
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
And for a Runge-Kutta table that claims order 3 (a real case from a public library):
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from physverify.butcher import check_tableau
|
|
85
|
+
check_tableau(A, b, c, claimed=3)
|
|
86
|
+
# FAIL: claims order 3 but satisfies the order conditions only up to 2 ...;
|
|
87
|
+
# linear order 3 is higher than the order 2: a convergence test on y' = lambda*y alone would overstate the order
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The second line explains why the library's own tests had not noticed: its test problem was the one kind of equation on which this method happens to look third order.
|
|
91
|
+
|
|
63
92
|
## Does it actually catch bugs?
|
|
64
93
|
|
|
65
94
|
To test it, 37 bugs were added one at a time to five small solvers (projectile, double pendulum, planet orbit, heat equation, shock tube):
|
|
@@ -74,16 +103,17 @@ The bugs and the checks were written by the same person, so these numbers show t
|
|
|
74
103
|
|
|
75
104
|
## Trying it on other people's code
|
|
76
105
|
|
|
77
|
-
To see what it does on code it was not written for, public Python physics repositories from GitHub were run in a locked-down container (October 2026): no network, the repository read-only, nothing from it run on the author's computer. 40 were drawn at random from topic searches (batches 1 to 4); another 60 were the newest small repositories whose README says the work is complete (batch 5), and 20 more were picked because they are quick to test (batch 6); batches 5 and 6 are not random.
|
|
106
|
+
To see what it does on code it was not written for, public Python physics repositories from GitHub were run in a locked-down container (October 2026): no network, the repository read-only, nothing from it run on the author's computer. 40 were drawn at random from topic searches (batches 1 to 4); another 60 were the newest small repositories whose README says the work is complete (batch 5), and 20 more were picked because they are quick to test (batch 6); batches 5 and 6 are not random. Batch 7 was a larger queue of 450 recently active small repositories from 61 topic searches, ordered by stars (not random either).
|
|
78
107
|
|
|
79
108
|
- **68 of the 120 could be tested.** Most of the others were not numerical solvers (neural networks, apps and games, analysis tools, finance), one needed a recent NVIDIA GPU, and a few need more set-up.
|
|
80
109
|
- **16 confirmed bugs in 12 of the 68.** For example: a measured energy that counts the magnetic-field term twice; backward time-stepping that is only first-order accurate; Runge-Kutta coefficient tables labelled with a higher order than they have; a spectral derivative that crashes for an odd number of grid points; boundary values applied after the linear solve instead of inside it; an insulated wall that leaks heat; a grid spacing off by one interval; a Kepler solver that fails for negative angles.
|
|
110
|
+
- **Batch 7: 182 of 450 tested, at least one confirmed defect in 41.** For example: a "Regge-Wheeler" ringdown code that uses the scalar-field potential; MHD shock jump ratios that break the Rankine-Hugoniot conditions; a reactor period for negative reactivity off by a factor of several thousand; a pressure solver that stops after one sweep; gravitational accelerations with the wrong sign; three files that do not parse. Some of the 41 are minor (a label, an optional viewer that crashes).
|
|
81
111
|
- **The rest had no confirmed bug.** Most passed every check that applied, often to 1e-13 or better.
|
|
82
112
|
- **Our own adapters were wrong about 28 times** before checking (a wrong constant, too-coarse steps, a wrong sign convention, and so on). Every FAIL was checked by hand before being called a bug, and these mistakes are recorded, not hidden.
|
|
83
113
|
|
|
84
114
|
These counts describe the repositories that were drawn; they are not a bug rate for physics code in general.
|
|
85
115
|
|
|
86
|
-
Each lesson became a new check or a clearer message in physverify. The testing and write-ups were done with help from an AI assistant; every number comes from running the code. Authors are told about a finding before their project is named, so the repositories are listed by ID: [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md).
|
|
116
|
+
Each lesson became a new check or a clearer message in physverify. The testing and write-ups were done with help from an AI assistant; every number comes from running the code. Authors are told about a finding before their project is named, so the repositories are listed by ID: [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md) (batches 1 to 6) and [results/corpus/BATCH7.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/BATCH7.md).
|
|
87
117
|
|
|
88
118
|
## Try it
|
|
89
119
|
|
|
@@ -105,6 +135,14 @@ You should see a list of checks, all passing. Now open `examples/damped_oscillat
|
|
|
105
135
|
|
|
106
136
|
To check your own code you write a short "spec" file. It tells physverify two things: the equations your code is meant to solve (written independently of your code), and how to run your code. [examples/verify_damped_oscillator.py](https://github.com/rcgiri-physics/physverify/blob/main/examples/verify_damped_oscillator.py) is a complete example to copy.
|
|
107
137
|
|
|
138
|
+
To start a spec for your own ODE solver, let physverify write one:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
physverify init verify_solver.py --solver mysolver --order 4
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Then fill in the two places marked TODO: the equations, written from the physics, and a starting state.
|
|
145
|
+
|
|
108
146
|
Other ways to run it:
|
|
109
147
|
|
|
110
148
|
```bash
|
|
@@ -113,6 +151,31 @@ physverify run my_spec.py --json - # machine-readable output
|
|
|
113
151
|
pytest my_spec.py # each check becomes a pytest test
|
|
114
152
|
```
|
|
115
153
|
|
|
154
|
+
## Run it on every push
|
|
155
|
+
|
|
156
|
+
Add physverify to your GitHub Actions workflow. The job fails when a check fails, just like a broken unit test:
|
|
157
|
+
|
|
158
|
+
```yaml
|
|
159
|
+
- uses: actions/checkout@v4
|
|
160
|
+
- uses: rcgiri-physics/physverify@v0.4.1
|
|
161
|
+
with:
|
|
162
|
+
specs: tests/verify_*.py
|
|
163
|
+
install: -e .
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The results table appears on the run's summary page. [docs/ci.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ci.md) covers the options, pre-commit and pytest.
|
|
167
|
+
|
|
168
|
+
## What's new in 0.4.0
|
|
169
|
+
|
|
170
|
+
- **Runge-Kutta tables:** besides the order, it now checks the "linear" order (what a test on y' = lambda*y measures), the stage order, A- and L-stability and stiff accuracy, and fails when the nodes `c` do not match the table.
|
|
171
|
+
- **Honest counts:** a check that cannot run because an optional package is missing now says SKIP instead of passing.
|
|
172
|
+
- **Time budget:** `physverify run --budget 60` stops a slow check from using up the whole run.
|
|
173
|
+
- **New helpers** for convergence studies that compare at the wrong time, for the coverage of error bars, and for ringdown frequencies; more exact reference solutions (`physverify references`).
|
|
174
|
+
- **`physverify smoke`:** a quick first pass that finds scripts that crash, hang on `input()` or produce NaNs.
|
|
175
|
+
- **Continuous integration:** a GitHub Action, `physverify init` to write a starter spec, and a pre-commit hook.
|
|
176
|
+
|
|
177
|
+
The full list is in [CHANGELOG.md](https://github.com/rcgiri-physics/physverify/blob/main/CHANGELOG.md).
|
|
178
|
+
|
|
116
179
|
## What it can and cannot do
|
|
117
180
|
|
|
118
181
|
- Works best for ordinary differential equations (orbits, pendulums, projectiles).
|
|
@@ -120,18 +183,37 @@ pytest my_spec.py # each check becomes a pytest tes
|
|
|
120
183
|
- Your code needs a way to accept an extra source term, or a small wrapper ("adapter") around it. The wrapper can itself contain mistakes.
|
|
121
184
|
- Passing all checks does not prove the code is correct, only that these tests found nothing wrong.
|
|
122
185
|
|
|
186
|
+
## Questions
|
|
187
|
+
|
|
188
|
+
**Do I have to change my code?** Usually not. Most codes can be checked through a short spec file that calls them. Codes with no way to add a source term need a small wrapper; [ROADMAP.md](https://github.com/rcgiri-physics/physverify/blob/main/ROADMAP.md) describes plans to need fewer of them.
|
|
189
|
+
|
|
190
|
+
**How long does it take?** The checks run your code several times at different resolutions, so the time is mostly your own code's run time. The example above takes about two seconds.
|
|
191
|
+
|
|
192
|
+
**What does INCONCLUSIVE mean?** The errors were too small or too noisy to measure an order (often they are already at round-off). The message says which, and what to try.
|
|
193
|
+
|
|
194
|
+
**Can it run untrusted code safely?** physverify itself runs code on your computer like any Python program. For other people's code, use the locked-down container in [sandbox/](https://github.com/rcgiri-physics/physverify/blob/main/sandbox/README.md).
|
|
195
|
+
|
|
196
|
+
## What's next
|
|
197
|
+
|
|
198
|
+
[ROADMAP.md](https://github.com/rcgiri-physics/physverify/blob/main/ROADMAP.md): checking code without writing a wrapper, 2D and 3D heat-equation checks, IMEX Runge-Kutta pairs and a documentation website with many examples.
|
|
199
|
+
|
|
123
200
|
## More
|
|
124
201
|
|
|
125
|
-
- [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md): the 120 public repositories, what was tested and what was found.
|
|
202
|
+
- [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md): the first 120 public repositories, what was tested and what was found; [BATCH7.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/BATCH7.md) for the next 450.
|
|
126
203
|
- [sandbox/README.md](https://github.com/rcgiri-physics/physverify/blob/main/sandbox/README.md): the locked-down container used to run other people's code.
|
|
127
204
|
- [results/own_work/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/own_work/SUMMARY.md): results on four of the author's own public physics projects (two real bugs found and fixed).
|
|
128
205
|
- [docs/technical.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/technical.md): full details, benchmark tables, limits and related work.
|
|
129
206
|
- [docs/patch.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/patch.md): what the exact "patch" checks assume.
|
|
130
207
|
- [docs/ising.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ising.md): checks for Monte Carlo codes.
|
|
208
|
+
- [docs/ci.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ci.md): `physverify init`, the GitHub Action, pre-commit and pytest.
|
|
131
209
|
- [docs/agent.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/agent.md): using it with an AI coding assistant.
|
|
132
210
|
- [CHANGELOG.md](https://github.com/rcgiri-physics/physverify/blob/main/CHANGELOG.md)
|
|
133
211
|
|
|
134
|
-
Run the project's own tests with `pip install -e .[dev]` then `python -m pytest -q` (about
|
|
212
|
+
Run the project's own tests with `pip install -e .[dev]` then `python -m pytest -q` (about 3 minutes).
|
|
213
|
+
|
|
214
|
+
## Citing
|
|
215
|
+
|
|
216
|
+
If physverify helps your work, please cite it: GitHub's "Cite this repository" button uses [CITATION.cff](https://github.com/rcgiri-physics/physverify/blob/main/CITATION.cff).
|
|
135
217
|
|
|
136
218
|
## Licence
|
|
137
219
|
|
|
@@ -2,7 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
**A tool that checks whether a physics simulation is solving the right equations correctly, not just running without errors.**
|
|
4
4
|
|
|
5
|
-
Status: early version (0.
|
|
5
|
+
Status: early version (0.4.0). It has been tried on 250 public physics repositories (see below), but nobody besides the author has used it yet, so feedback is very welcome.
|
|
6
|
+
|
|
7
|
+
## Who it is for
|
|
8
|
+
|
|
9
|
+
- Students and researchers who wrote their own solver (orbits, pendulums, heat flow, shock tubes, Monte Carlo) and want more confidence than "the plot looks right".
|
|
10
|
+
- Maintainers of numerical libraries who want their methods checked on every push, for example that each Runge-Kutta table really has the order its name says.
|
|
11
|
+
- Anyone reviewing someone else's simulation code.
|
|
12
|
+
|
|
13
|
+
You do not need to know numerical analysis to read the results: each check says in plain words what it measured and what it expected.
|
|
6
14
|
|
|
7
15
|
## The problem it addresses
|
|
8
16
|
|
|
@@ -19,10 +27,31 @@ physverify uses a few standard ideas from numerical analysis:
|
|
|
19
27
|
5. **Exact "patch" tests.** Some solutions should be reproduced to computer precision. This catches small bugs that the other checks let through.
|
|
20
28
|
6. **Algebraic checks on the method itself.** For a Runge-Kutta method, whether the coefficients really have the order they claim can be settled exactly, with no test problem (the "order conditions").
|
|
21
29
|
|
|
22
|
-
Each check reports **PASS**, **FAIL**, **EXACT** or **INCONCLUSIVE
|
|
30
|
+
Each check reports **PASS**, **FAIL**, **EXACT** or **INCONCLUSIVE** (or **SKIP** when an optional dependency is missing).
|
|
23
31
|
|
|
24
32
|
There is also a separate set of checks for random (Monte Carlo) simulations, using the 2D Ising model as the example.
|
|
25
33
|
|
|
34
|
+
### What a result looks like
|
|
35
|
+
|
|
36
|
+
```text
|
|
37
|
+
mms PASS midpoint damped oscillator: observed order 2.04 (expected 2)
|
|
38
|
+
reference PASS midpoint damped oscillator: observed order 2.01 (expected 2)
|
|
39
|
+
mms PASS midpoint damped oscillator (whole trajectory): observed order 2.02 (expected 2)
|
|
40
|
+
patch EXACT midpoint damped oscillator: reproduced to round-off (largest error 2.7e-15), as designed
|
|
41
|
+
4 of 4 checks passed
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
And for a Runge-Kutta table that claims order 3 (a real case from a public library):
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from physverify.butcher import check_tableau
|
|
48
|
+
check_tableau(A, b, c, claimed=3)
|
|
49
|
+
# FAIL: claims order 3 but satisfies the order conditions only up to 2 ...;
|
|
50
|
+
# linear order 3 is higher than the order 2: a convergence test on y' = lambda*y alone would overstate the order
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The second line explains why the library's own tests had not noticed: its test problem was the one kind of equation on which this method happens to look third order.
|
|
54
|
+
|
|
26
55
|
## Does it actually catch bugs?
|
|
27
56
|
|
|
28
57
|
To test it, 37 bugs were added one at a time to five small solvers (projectile, double pendulum, planet orbit, heat equation, shock tube):
|
|
@@ -37,16 +66,17 @@ The bugs and the checks were written by the same person, so these numbers show t
|
|
|
37
66
|
|
|
38
67
|
## Trying it on other people's code
|
|
39
68
|
|
|
40
|
-
To see what it does on code it was not written for, public Python physics repositories from GitHub were run in a locked-down container (October 2026): no network, the repository read-only, nothing from it run on the author's computer. 40 were drawn at random from topic searches (batches 1 to 4); another 60 were the newest small repositories whose README says the work is complete (batch 5), and 20 more were picked because they are quick to test (batch 6); batches 5 and 6 are not random.
|
|
69
|
+
To see what it does on code it was not written for, public Python physics repositories from GitHub were run in a locked-down container (October 2026): no network, the repository read-only, nothing from it run on the author's computer. 40 were drawn at random from topic searches (batches 1 to 4); another 60 were the newest small repositories whose README says the work is complete (batch 5), and 20 more were picked because they are quick to test (batch 6); batches 5 and 6 are not random. Batch 7 was a larger queue of 450 recently active small repositories from 61 topic searches, ordered by stars (not random either).
|
|
41
70
|
|
|
42
71
|
- **68 of the 120 could be tested.** Most of the others were not numerical solvers (neural networks, apps and games, analysis tools, finance), one needed a recent NVIDIA GPU, and a few need more set-up.
|
|
43
72
|
- **16 confirmed bugs in 12 of the 68.** For example: a measured energy that counts the magnetic-field term twice; backward time-stepping that is only first-order accurate; Runge-Kutta coefficient tables labelled with a higher order than they have; a spectral derivative that crashes for an odd number of grid points; boundary values applied after the linear solve instead of inside it; an insulated wall that leaks heat; a grid spacing off by one interval; a Kepler solver that fails for negative angles.
|
|
73
|
+
- **Batch 7: 182 of 450 tested, at least one confirmed defect in 41.** For example: a "Regge-Wheeler" ringdown code that uses the scalar-field potential; MHD shock jump ratios that break the Rankine-Hugoniot conditions; a reactor period for negative reactivity off by a factor of several thousand; a pressure solver that stops after one sweep; gravitational accelerations with the wrong sign; three files that do not parse. Some of the 41 are minor (a label, an optional viewer that crashes).
|
|
44
74
|
- **The rest had no confirmed bug.** Most passed every check that applied, often to 1e-13 or better.
|
|
45
75
|
- **Our own adapters were wrong about 28 times** before checking (a wrong constant, too-coarse steps, a wrong sign convention, and so on). Every FAIL was checked by hand before being called a bug, and these mistakes are recorded, not hidden.
|
|
46
76
|
|
|
47
77
|
These counts describe the repositories that were drawn; they are not a bug rate for physics code in general.
|
|
48
78
|
|
|
49
|
-
Each lesson became a new check or a clearer message in physverify. The testing and write-ups were done with help from an AI assistant; every number comes from running the code. Authors are told about a finding before their project is named, so the repositories are listed by ID: [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md).
|
|
79
|
+
Each lesson became a new check or a clearer message in physverify. The testing and write-ups were done with help from an AI assistant; every number comes from running the code. Authors are told about a finding before their project is named, so the repositories are listed by ID: [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md) (batches 1 to 6) and [results/corpus/BATCH7.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/BATCH7.md).
|
|
50
80
|
|
|
51
81
|
## Try it
|
|
52
82
|
|
|
@@ -68,6 +98,14 @@ You should see a list of checks, all passing. Now open `examples/damped_oscillat
|
|
|
68
98
|
|
|
69
99
|
To check your own code you write a short "spec" file. It tells physverify two things: the equations your code is meant to solve (written independently of your code), and how to run your code. [examples/verify_damped_oscillator.py](https://github.com/rcgiri-physics/physverify/blob/main/examples/verify_damped_oscillator.py) is a complete example to copy.
|
|
70
100
|
|
|
101
|
+
To start a spec for your own ODE solver, let physverify write one:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
physverify init verify_solver.py --solver mysolver --order 4
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Then fill in the two places marked TODO: the equations, written from the physics, and a starting state.
|
|
108
|
+
|
|
71
109
|
Other ways to run it:
|
|
72
110
|
|
|
73
111
|
```bash
|
|
@@ -76,6 +114,31 @@ physverify run my_spec.py --json - # machine-readable output
|
|
|
76
114
|
pytest my_spec.py # each check becomes a pytest test
|
|
77
115
|
```
|
|
78
116
|
|
|
117
|
+
## Run it on every push
|
|
118
|
+
|
|
119
|
+
Add physverify to your GitHub Actions workflow. The job fails when a check fails, just like a broken unit test:
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
- uses: actions/checkout@v4
|
|
123
|
+
- uses: rcgiri-physics/physverify@v0.4.1
|
|
124
|
+
with:
|
|
125
|
+
specs: tests/verify_*.py
|
|
126
|
+
install: -e .
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The results table appears on the run's summary page. [docs/ci.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ci.md) covers the options, pre-commit and pytest.
|
|
130
|
+
|
|
131
|
+
## What's new in 0.4.0
|
|
132
|
+
|
|
133
|
+
- **Runge-Kutta tables:** besides the order, it now checks the "linear" order (what a test on y' = lambda*y measures), the stage order, A- and L-stability and stiff accuracy, and fails when the nodes `c` do not match the table.
|
|
134
|
+
- **Honest counts:** a check that cannot run because an optional package is missing now says SKIP instead of passing.
|
|
135
|
+
- **Time budget:** `physverify run --budget 60` stops a slow check from using up the whole run.
|
|
136
|
+
- **New helpers** for convergence studies that compare at the wrong time, for the coverage of error bars, and for ringdown frequencies; more exact reference solutions (`physverify references`).
|
|
137
|
+
- **`physverify smoke`:** a quick first pass that finds scripts that crash, hang on `input()` or produce NaNs.
|
|
138
|
+
- **Continuous integration:** a GitHub Action, `physverify init` to write a starter spec, and a pre-commit hook.
|
|
139
|
+
|
|
140
|
+
The full list is in [CHANGELOG.md](https://github.com/rcgiri-physics/physverify/blob/main/CHANGELOG.md).
|
|
141
|
+
|
|
79
142
|
## What it can and cannot do
|
|
80
143
|
|
|
81
144
|
- Works best for ordinary differential equations (orbits, pendulums, projectiles).
|
|
@@ -83,18 +146,37 @@ pytest my_spec.py # each check becomes a pytest tes
|
|
|
83
146
|
- Your code needs a way to accept an extra source term, or a small wrapper ("adapter") around it. The wrapper can itself contain mistakes.
|
|
84
147
|
- Passing all checks does not prove the code is correct, only that these tests found nothing wrong.
|
|
85
148
|
|
|
149
|
+
## Questions
|
|
150
|
+
|
|
151
|
+
**Do I have to change my code?** Usually not. Most codes can be checked through a short spec file that calls them. Codes with no way to add a source term need a small wrapper; [ROADMAP.md](https://github.com/rcgiri-physics/physverify/blob/main/ROADMAP.md) describes plans to need fewer of them.
|
|
152
|
+
|
|
153
|
+
**How long does it take?** The checks run your code several times at different resolutions, so the time is mostly your own code's run time. The example above takes about two seconds.
|
|
154
|
+
|
|
155
|
+
**What does INCONCLUSIVE mean?** The errors were too small or too noisy to measure an order (often they are already at round-off). The message says which, and what to try.
|
|
156
|
+
|
|
157
|
+
**Can it run untrusted code safely?** physverify itself runs code on your computer like any Python program. For other people's code, use the locked-down container in [sandbox/](https://github.com/rcgiri-physics/physverify/blob/main/sandbox/README.md).
|
|
158
|
+
|
|
159
|
+
## What's next
|
|
160
|
+
|
|
161
|
+
[ROADMAP.md](https://github.com/rcgiri-physics/physverify/blob/main/ROADMAP.md): checking code without writing a wrapper, 2D and 3D heat-equation checks, IMEX Runge-Kutta pairs and a documentation website with many examples.
|
|
162
|
+
|
|
86
163
|
## More
|
|
87
164
|
|
|
88
|
-
- [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md): the 120 public repositories, what was tested and what was found.
|
|
165
|
+
- [results/corpus/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/SUMMARY.md): the first 120 public repositories, what was tested and what was found; [BATCH7.md](https://github.com/rcgiri-physics/physverify/blob/main/results/corpus/BATCH7.md) for the next 450.
|
|
89
166
|
- [sandbox/README.md](https://github.com/rcgiri-physics/physverify/blob/main/sandbox/README.md): the locked-down container used to run other people's code.
|
|
90
167
|
- [results/own_work/SUMMARY.md](https://github.com/rcgiri-physics/physverify/blob/main/results/own_work/SUMMARY.md): results on four of the author's own public physics projects (two real bugs found and fixed).
|
|
91
168
|
- [docs/technical.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/technical.md): full details, benchmark tables, limits and related work.
|
|
92
169
|
- [docs/patch.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/patch.md): what the exact "patch" checks assume.
|
|
93
170
|
- [docs/ising.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ising.md): checks for Monte Carlo codes.
|
|
171
|
+
- [docs/ci.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/ci.md): `physverify init`, the GitHub Action, pre-commit and pytest.
|
|
94
172
|
- [docs/agent.md](https://github.com/rcgiri-physics/physverify/blob/main/docs/agent.md): using it with an AI coding assistant.
|
|
95
173
|
- [CHANGELOG.md](https://github.com/rcgiri-physics/physverify/blob/main/CHANGELOG.md)
|
|
96
174
|
|
|
97
|
-
Run the project's own tests with `pip install -e .[dev]` then `python -m pytest -q` (about
|
|
175
|
+
Run the project's own tests with `pip install -e .[dev]` then `python -m pytest -q` (about 3 minutes).
|
|
176
|
+
|
|
177
|
+
## Citing
|
|
178
|
+
|
|
179
|
+
If physverify helps your work, please cite it: GitHub's "Cite this repository" button uses [CITATION.cff](https://github.com/rcgiri-physics/physverify/blob/main/CITATION.cff).
|
|
98
180
|
|
|
99
181
|
## Licence
|
|
100
182
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "physverify"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Verification checks for physics simulation code: manufactured solutions, order-of-accuracy studies and a seeded-bug benchmark."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
"""physverify: verification checks for physics simulation code."""
|
|
2
2
|
|
|
3
|
-
from .checks import Check, OrderResult, Result
|
|
3
|
+
from .checks import Check, OrderResult, Result, SkipCheck, requires
|
|
4
4
|
from .mms import ManufacturedODE, mms_order_study, zero_source
|
|
5
5
|
from .order import check_order, fit_order, pairwise_orders
|
|
6
6
|
from .order import check_exact
|
|
7
7
|
from .studies import invariant_study, ode_checks, patch_study, reference_study
|
|
8
8
|
|
|
9
|
-
__version__ = "0.
|
|
9
|
+
__version__ = "0.4.1"
|
|
10
10
|
|
|
11
11
|
__all__ = [
|
|
12
12
|
"Check",
|
|
13
13
|
"ManufacturedODE",
|
|
14
14
|
"OrderResult",
|
|
15
15
|
"Result",
|
|
16
|
+
"SkipCheck",
|
|
16
17
|
"check_exact",
|
|
17
18
|
"check_order",
|
|
18
19
|
"fit_order",
|
|
@@ -20,6 +21,7 @@ __all__ = [
|
|
|
20
21
|
"mms_order_study",
|
|
21
22
|
"ode_checks",
|
|
22
23
|
"pairwise_orders",
|
|
24
|
+
"requires",
|
|
23
25
|
"patch_study",
|
|
24
26
|
"reference_study",
|
|
25
27
|
"zero_source",
|
|
@@ -150,7 +150,7 @@ TOOLS = [
|
|
|
150
150
|
"invariants, limits, exactness) on physics simulation code. Takes spec files that "
|
|
151
151
|
"define checks(); with no specs given, finds verify_*.py files under `project`. "
|
|
152
152
|
"Returns a JSON report: per check family, label, status (PASS, EXACT, FAIL, "
|
|
153
|
-
"INCONCLUSIVE, ERROR), message, observed and expected order, step sizes and errors."
|
|
153
|
+
"INCONCLUSIVE, ERROR, SKIP), message, observed and expected order, step sizes and errors."
|
|
154
154
|
),
|
|
155
155
|
"inputSchema": {
|
|
156
156
|
"type": "object",
|