ulpwise 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ulpwise-0.2.0 → ulpwise-0.3.0}/.github/workflows/release.yml +19 -1
- {ulpwise-0.2.0 → ulpwise-0.3.0}/CHANGELOG.md +31 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/Cargo.lock +1 -1
- {ulpwise-0.2.0 → ulpwise-0.3.0}/Cargo.toml +1 -1
- {ulpwise-0.2.0 → ulpwise-0.3.0}/PKG-INFO +91 -12
- {ulpwise-0.2.0 → ulpwise-0.3.0}/README.md +90 -11
- {ulpwise-0.2.0 → ulpwise-0.3.0}/pyproject.toml +1 -1
- ulpwise-0.3.0/python/ulpwise/__init__.py +340 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/__main__.py +36 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/corpus/__init__.py +36 -1
- {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/corpus/cases.json +2 -1
- {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/plugin.py +4 -3
- ulpwise-0.3.0/python/ulpwise/scan.py +622 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/tests/test_core.py +95 -5
- ulpwise-0.3.0/tests/test_corpus.py +77 -0
- ulpwise-0.3.0/tests/test_scan.py +191 -0
- ulpwise-0.2.0/python/ulpwise/__init__.py +0 -184
- ulpwise-0.2.0/tests/test_corpus.py +0 -41
- {ulpwise-0.2.0 → ulpwise-0.3.0}/.github/workflows/ci.yml +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/.gitignore +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/CONTRIBUTING.md +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/LICENSE-APACHE +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/LICENSE-MIT +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/examples/sqrt_knife_values.py +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/examples/torch_sqrt_conformance.py +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/survey.py +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/edge.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/exact.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/knife.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/lib.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/py.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/src/ulp.rs +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/README.md +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/results.csv +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/results.md +0 -0
- {ulpwise-0.2.0 → ulpwise-0.3.0}/tests/test_survey.py +0 -0
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
# uploads them to PyPI through trusted publishing (no API token stored anywhere):
|
|
3
3
|
# the PyPI project "ulpwise" must list this repository, this workflow file name and the
|
|
4
4
|
# "pypi" environment as its publisher. A manual run only builds and checks the wheels,
|
|
5
|
-
# unless its "publish" input is set, in which case it also uploads
|
|
5
|
+
# unless its "publish" input is set, in which case it also uploads, tags the commit and
|
|
6
|
+
# creates the GitHub release with the CHANGELOG section as notes and the wheels attached.
|
|
6
7
|
name: release
|
|
7
8
|
|
|
8
9
|
on:
|
|
@@ -119,3 +120,20 @@ jobs:
|
|
|
119
120
|
run: |
|
|
120
121
|
git tag "v${{ steps.version.outputs.version }}"
|
|
121
122
|
git push origin "v${{ steps.version.outputs.version }}"
|
|
123
|
+
- name: GitHub release with the CHANGELOG section as notes and the wheels attached
|
|
124
|
+
env:
|
|
125
|
+
GH_TOKEN: ${{ github.token }}
|
|
126
|
+
VERSION: ${{ steps.version.outputs.version }}
|
|
127
|
+
run: |
|
|
128
|
+
python3 - > notes.md <<'EOF'
|
|
129
|
+
import os, re
|
|
130
|
+
version = os.environ["VERSION"]
|
|
131
|
+
text = open("CHANGELOG.md", encoding="utf-8").read()
|
|
132
|
+
m = re.search(r"^## " + re.escape(version) + r"\b.*?\n(.*?)(?=^## |\Z)", text, re.M | re.S)
|
|
133
|
+
print(m.group(1).strip() if m else f"ulpwise {version}, see CHANGELOG.md")
|
|
134
|
+
EOF
|
|
135
|
+
if gh release view "v$VERSION" > /dev/null 2>&1; then
|
|
136
|
+
gh release upload "v$VERSION" dist/* --clobber
|
|
137
|
+
else
|
|
138
|
+
gh release create "v$VERSION" dist/* --title "ulpwise $VERSION" --notes-file notes.md
|
|
139
|
+
fi
|
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0 (2026-09-26)
|
|
4
|
+
|
|
5
|
+
- `ulpwise scan`: static scan of a repository (a directory, a GitHub URL or `owner/repo`, cloned
|
|
6
|
+
with depth 1) for the floating point patterns behind the corpus bugs. Twelve rules with severity,
|
|
7
|
+
reason, replacement and upstream example: `exp-of-square`, `sin-of-pi-times`, `softplus-by-hand`,
|
|
8
|
+
`logsumexp-by-hand`, `hypot-by-hand`, `sqrt-of-difference`, `one-minus-cos`, `log1p-by-hand`,
|
|
9
|
+
`expm1-by-hand`, `atan-of-quotient`, `small-angle-division`, `acos-for-angle`. Findings carry
|
|
10
|
+
file, line, function and the source line; the report ends with the functions that do the most
|
|
11
|
+
elementary math. `--report` writes Markdown, `--rules` filters, `--fail-on` gates CI,
|
|
12
|
+
`--include-tests` widens the walk. On kornia main the four `small-angle-division` and
|
|
13
|
+
`one-minus-cos` lines are the ones kornia #4897 fixes.
|
|
14
|
+
- `ulpwise scan --run`: imports the math-heavy module level functions and calls them on the same
|
|
15
|
+
grid in float32 and float64, reporting the largest error in ulps of the largest output and the
|
|
16
|
+
worst elementwise ulp distance with its input. Static methods run, instance methods and other
|
|
17
|
+
skips carry their reason. `--run-limit` bounds it, `--run-installed` imports the installed package
|
|
18
|
+
instead of the scanned tree.
|
|
19
|
+
- `ulp_distance`, `ulp_distances`, `ordered`, `max_ulp` and `assert_max_ulp` take `f16` and
|
|
20
|
+
`bf16`, and `flatten` reads the dtype off float16 and bfloat16 numpy arrays and torch tensors.
|
|
21
|
+
Inputs that are not representable are rounded to nearest even first, so a bfloat16 tensor can be
|
|
22
|
+
measured against a float64 reference; without an explicit dtype the less precise of the two
|
|
23
|
+
inputs decides. Cross-checked against the numpy int16 view for float16 and the torch view for
|
|
24
|
+
bfloat16.
|
|
25
|
+
- `special("f16")` and `special("bf16")`: the same 29 named edge values as f32 and f64, so
|
|
26
|
+
`edge_values`, the `edge_f16` and `edge_bf16` pytest fixtures and `ulpwise special f16` work.
|
|
27
|
+
Every value satisfies the same checks as the f32 and f64 tables, run through numpy for float16
|
|
28
|
+
and torch for bfloat16.
|
|
29
|
+
- `ulpwise corpus`: runs the regression corpus against the installed packages without pytest and
|
|
30
|
+
prints one line per case, present, fixed or skipped, with the installed version and the upstream
|
|
31
|
+
reference. `--repo` filters by repository or case id, `--fail-if-present` makes a present bug exit 1.
|
|
32
|
+
Any exception from a repro counts as present, like the pytest run: several corpus bugs are crashes.
|
|
33
|
+
|
|
3
34
|
## 0.2.0 (2026-09-26)
|
|
4
35
|
|
|
5
36
|
- `ulpwise survey` (`ulpwise.survey`): accuracy survey of 61 elementary and special functions of
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ulpwise
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Framework :: Pytest
|
|
6
6
|
Classifier: Intended Audience :: Developers
|
|
@@ -30,7 +30,7 @@ Project-URL: Repository, https://github.com/Nicholas022400701/ulpwise
|
|
|
30
30
|
|
|
31
31
|
Numerical conformance testing for ML code. A Rust core with a Python API and a pytest plugin.
|
|
32
32
|
|
|
33
|
-
`ulpwise` answers
|
|
33
|
+
`ulpwise` answers five questions that come up every time a numerical test goes red on one
|
|
34
34
|
platform and green on another:
|
|
35
35
|
|
|
36
36
|
1. **Which inputs break first?** Named edge values computed from the float format (the largest
|
|
@@ -47,6 +47,10 @@ platform and green on another:
|
|
|
47
47
|
`ulpwise survey` measures 61 elementary and special functions of torch, numpy, scipy and jax in
|
|
48
48
|
ulps against a 200 bit mpmath reference and, for torch, checks every error against the tolerance
|
|
49
49
|
of torch's own `OpInfo` reference test, default and per op override.
|
|
50
|
+
5. **Where should I read first in a repository I do not know?** `ulpwise scan` parses every Python
|
|
51
|
+
file and reports the expressions behind the bugs in the corpus (`exp(x * x)`, `sin(pi * x)`,
|
|
52
|
+
`1 - cos(x)`, `sqrt(a * a + b * b)`, `log(1 + exp(x))`, divisions by an unguarded angle, ...)
|
|
53
|
+
with file, line and function, then lists the functions with the most elementary math.
|
|
50
54
|
|
|
51
55
|
## Why this exists
|
|
52
56
|
|
|
@@ -109,12 +113,15 @@ The Rust crate is usable on its own (`cargo add --git https://github.com/Nichola
|
|
|
109
113
|
```python
|
|
110
114
|
import ulpwise
|
|
111
115
|
|
|
112
|
-
# ulp distances, dtype aware: a float32 tensor is measured in float32 ulps
|
|
116
|
+
# ulp distances, dtype aware: a float32 tensor is measured in float32 ulps, a bfloat16 tensor
|
|
117
|
+
# against a float64 reference in bfloat16 ulps. f64, f32, f16 and bf16.
|
|
113
118
|
ulpwise.ulp_distance(0.9235056042671204, 0.9235056638717651, "f32") # 1
|
|
119
|
+
ulpwise.ulp_distance(1.0, 1.001, "bf16") # 0, both round to 1.0
|
|
114
120
|
ulpwise.assert_max_ulp(torch_out, reference, max_ulp_=2) # floats, lists, numpy, torch
|
|
115
121
|
|
|
116
|
-
# the 29 named edge values of a dtype
|
|
122
|
+
# the 29 named edge values of a dtype, f64 f32 f16 bf16
|
|
117
123
|
dict(ulpwise.special("f32"))["square_underflows_to_zero"] # 2.6469779601696886e-23, the largest x with x * x == 0
|
|
124
|
+
dict(ulpwise.special("bf16"))["square_overflows"] # 1.8446744073709552e+19, 2 ** 64
|
|
118
125
|
|
|
119
126
|
# exact placement of a result relative to its rounding midpoint
|
|
120
127
|
ulpwise.midpoint("sqrt", 0.8528626561164856, "f32") # (0.9235056042671204, 0.000377, True, False)
|
|
@@ -133,8 +140,8 @@ ulpwise.sqrt_cr(0.8528626561164856, "f32") # 0.9235056042671204
|
|
|
133
140
|
|
|
134
141
|
### pytest plugin
|
|
135
142
|
|
|
136
|
-
Installed automatically. A test that takes `edge_f32` or `
|
|
137
|
-
value, and `assert_max_ulp` is available as a fixture:
|
|
143
|
+
Installed automatically. A test that takes `edge_f64`, `edge_f32`, `edge_f16` or `edge_bf16` runs
|
|
144
|
+
once per named edge value, and `assert_max_ulp` is available as a fixture:
|
|
138
145
|
|
|
139
146
|
```python
|
|
140
147
|
def test_my_kernel_survives_the_edges(edge_f32, assert_max_ulp):
|
|
@@ -151,6 +158,9 @@ project grew out of, each with a runnable repro, the buggy behaviour quoted from
|
|
|
151
158
|
request, and the check that tells the two apart. `pytest tests/test_corpus.py` runs every case
|
|
152
159
|
whose packages are installed; a case is an expected failure while the installed release is not
|
|
153
160
|
known to contain the fix, so the run tells you which bugs are present in your environment.
|
|
161
|
+
`ulpwise corpus` does the same without pytest, one line per case with the installed version and
|
|
162
|
+
`present`, `fixed` or `skipped`; `ulpwise corpus --repo pytorch --fail-if-present` is the CI gate
|
|
163
|
+
form.
|
|
154
164
|
|
|
155
165
|
| case | kind | merged |
|
|
156
166
|
|---|---|---|
|
|
@@ -181,6 +191,69 @@ expected failures until a release contains the fix (`fixed_in_release` in `cases
|
|
|
181
191
|
in a temporary directory and needs no weights; two more ultralytics fixes (#26240, #26246) are not in
|
|
182
192
|
the corpus yet because their repros need model weights or the COCO evaluator.
|
|
183
193
|
|
|
194
|
+
## Repository scan
|
|
195
|
+
|
|
196
|
+
```sh
|
|
197
|
+
ulpwise scan kornia/kornia --report kornia.md # clone with depth 1, write a Markdown report
|
|
198
|
+
ulpwise scan . --fail-on high # CI gate: exit 1 on a high severity finding
|
|
199
|
+
ulpwise scan path/to/repo --rules one-minus-cos,small-angle-division --top 40
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
The scan is static and needs nothing installed: it parses each file with `ast`, walks every
|
|
203
|
+
function and matches twelve patterns, each with a severity, the reason it loses digits or
|
|
204
|
+
overflows, the usual replacement and, where one exists, the upstream bug it comes from.
|
|
205
|
+
|
|
206
|
+
| rule | severity | pattern |
|
|
207
|
+
|---|---|---|
|
|
208
|
+
| `exp-of-square` | high | `exp(x * x)`, `exp(x ** 2)`, `(-x.pow(2)).exp()`: the rounding error of the square is multiplied by `x * x / 2` ulps (pytorch #198664) |
|
|
209
|
+
| `sin-of-pi-times` | high | `sin(pi * x)`, `cos(pi * x)`: the product is rounded before the argument reduction (pytorch #198663) |
|
|
210
|
+
| `softplus-by-hand` | high | `log(1 + exp(x))` |
|
|
211
|
+
| `logsumexp-by-hand` | high | `log(exp(a) + exp(b))`, `log(sum(exp(x)))` |
|
|
212
|
+
| `hypot-by-hand` | high | `sqrt(a * a + b * b)` |
|
|
213
|
+
| `sqrt-of-difference` | medium | `sqrt(a - b)` |
|
|
214
|
+
| `one-minus-cos` | medium | `1 - cos(x)` (kornia #4897) |
|
|
215
|
+
| `log1p-by-hand`, `expm1-by-hand` | medium | `log(1 + x)`, `exp(x) - 1` |
|
|
216
|
+
| `atan-of-quotient` | medium | `atan(y / x)` |
|
|
217
|
+
| `small-angle-division` | medium | `/ theta`, `/ theta ** 2`, `/ sin(theta)` in a function that takes `sin` or `cos` of `theta` and has no `where`, `clamp`, `eps` or series in sight (kornia #4838, #4897) |
|
|
218
|
+
| `acos-for-angle` | info | `acos`, `asin` used to recover an angle |
|
|
219
|
+
|
|
220
|
+
On kornia `main` at `e05b0ee` the scan takes 4 s for 506 files and reports 35 findings. The
|
|
221
|
+
`one-minus-cos` and `small-angle-division` findings are the four lines of `So3.right_jacobian` and
|
|
222
|
+
`So3.left_jacobian` that kornia #4897 fixes, `So3.log` (kornia #4838) is under `acos-for-angle`,
|
|
223
|
+
and `Se3.exp` (also #4897) is under `one-minus-cos`. The scan puts `ellipse_to_laf` (kornia #4768)
|
|
224
|
+
on the list too, for a `sqrt` of a difference; the bug there was a different one, so that entry is
|
|
225
|
+
what the scan is: a reading list, not a verdict.
|
|
226
|
+
|
|
227
|
+
`--run` adds the dynamic half. The module level functions with the most elementary math are
|
|
228
|
+
imported and called with the same 91 point grid (both signs of `1e-8` to `1e3`, and zero) for every
|
|
229
|
+
required argument, in float32 and in float64, torch first and numpy second, and the float32 result
|
|
230
|
+
is measured against the float64 one. Two numbers per function: `at scale`, the largest absolute
|
|
231
|
+
error in ulps of the largest output, and `elementwise`, the worst per element ulp distance with
|
|
232
|
+
the input where it happens. Read both. A rotation matrix has entries that should be zero, and
|
|
233
|
+
there the elementwise count compares float32 rounding noise with float64 rounding noise and reaches
|
|
234
|
+
`1e9` while the matrix is fine to one ulp at scale. A function whose output spans forty orders of
|
|
235
|
+
magnitude, a Bessel function, has a meaningless `at scale` number and a meaningful elementwise one.
|
|
236
|
+
A small angle formula without a guard, `(1 - cos(theta)) / theta ** 2`, shows `9e6` in both
|
|
237
|
+
columns. Static methods run; instance methods, functions that need other arguments, fail to import
|
|
238
|
+
or return something that is not a float array are counted with the reason and skipped, never
|
|
239
|
+
guessed at. This imports and runs the repository's code, from the scanned tree, or from the
|
|
240
|
+
package installed in the environment with `--run-installed` when the tree has unbuilt extensions.
|
|
241
|
+
|
|
242
|
+
```sh
|
|
243
|
+
ulpwise scan kornia/kornia --run --run-limit 200 --report kornia.md
|
|
244
|
+
ulpwise scan pytorch/vision --run --run-installed --run-limit 150
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
On kornia main 9 of the 200 busiest functions run as is, the rest are instance methods or want
|
|
248
|
+
shaped inputs. `adjust_log` is on both lists: the static scan flags its `(1 + image).log2()` as
|
|
249
|
+
`log1p-by-hand`, and the run shows `8.7e8` elementwise ulps at `x = 5.6e-8` next to `0.7` ulps at
|
|
250
|
+
scale, which is the right reading for a function defined on images in `[0, 1]`.
|
|
251
|
+
|
|
252
|
+
In ML repositories most `high` findings are learning rate schedules (`cos(pi * progress)` with
|
|
253
|
+
`progress` in `[0, 1]`) and pixel distances (`sqrt(dx * dx + dy * dy)` on coordinates below
|
|
254
|
+
`1e4`), where the argument is bounded and the pattern is harmless. The scan cannot know the bound;
|
|
255
|
+
the reading list is where that judgement happens.
|
|
256
|
+
|
|
184
257
|
## Accuracy survey
|
|
185
258
|
|
|
186
259
|
```sh
|
|
@@ -198,14 +271,18 @@ finite mismatches and the worst input. For torch it also reports how many inputs
|
|
|
198
271
|
kernel and the scalar tail disagree on, and how many inputs would fail torch's reference test under
|
|
199
272
|
the dtype default tolerance and under the op's `OpInfo` override, read from `op_db`.
|
|
200
273
|
|
|
201
|
-
[`studies/accuracy-survey-2026-09`](studies/accuracy-survey-2026-09/README.md) is the first run
|
|
274
|
+
[`studies/accuracy-survey-2026-09`](https://github.com/Nicholas022400701/ulpwise/blob/main/studies/accuracy-survey-2026-09/README.md) is the first run
|
|
202
275
|
(torch 2.14.0+cpu, numpy 2.2.6, scipy 1.18.1, jax 0.11.2, Linux x86_64 AVX512). The short version:
|
|
203
276
|
|
|
204
277
|
- torch's `bessel_j0/j1/y0/y1` and `airy_ai` in float64 are off by 2.6e9 to 3.9e12 ulps and the
|
|
205
|
-
`precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
|
|
278
|
+
`precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
|
|
279
|
+
(pytorch #198583).
|
|
206
280
|
- torch's `polygamma(1, x)` in float64 keeps about 9 digits (4.0e6 ulps, 46 percent of inputs
|
|
207
281
|
beyond 10 ulps) and passes the default float64 tolerance, which at `rtol = atol = 1e-7` tolerates
|
|
208
|
-
about 4.5e8 ulps.
|
|
282
|
+
about 4.5e8 ulps; in float32 it loses every digit for large negative `x` (pytorch #198663).
|
|
283
|
+
- torch's `erfcx` for negative `x` is off by up to `x*x/2` ulps, 44 in float32 at `x = -8.44` and
|
|
284
|
+
157 in float64 at `x = -23.25`, and scipy's float64 `erfcx` returns the same wrong values
|
|
285
|
+
(pytorch #198664).
|
|
209
286
|
- for 12 of 61 torch functions the AVX512 kernel and the scalar tail return different floats for
|
|
210
287
|
the same input, up to 246 of 619 inputs for `mish`.
|
|
211
288
|
- jax on CPU flushes subnormals to zero, its float64 `erfinv` loses 5 digits near the ends of the
|
|
@@ -226,9 +303,11 @@ cross-checks both against `fractions.Fraction` and `decimal.Decimal` at 80 digit
|
|
|
226
303
|
|
|
227
304
|
## Roadmap
|
|
228
305
|
|
|
306
|
+
- `ulpwise scan --run` for methods and functions with tensor shape requirements, by reading the
|
|
307
|
+
docstring and the checks for the shapes, and an mpmath reference for the scalar functions.
|
|
229
308
|
- Mutation scoring for numerical tests: single token mutants of the code under test (`abs`, a
|
|
230
309
|
dropped `sqrt`, `/ 4` for `/ 16`) run against the test suite, reporting which survive.
|
|
231
|
-
- `float16` and `bfloat16`
|
|
310
|
+
- `float16` and `bfloat16` `spacing`, `next_up`, knife edges and exact oracles (ulp distances and edge values are done).
|
|
232
311
|
- Exact references for transcendental functions in Rust (correctly rounded `exp`, `log`, ...) so
|
|
233
312
|
the `f64` knife-edge scans do not need mpmath.
|
|
234
313
|
- Survey backends for CUDA and MPS, and `float16` / `bfloat16` rows.
|
|
@@ -246,8 +325,8 @@ those pull requests carries the same disclosure.
|
|
|
246
325
|
|
|
247
326
|
Licensed under either of
|
|
248
327
|
|
|
249
|
-
- the MIT license ([LICENSE-MIT](LICENSE-MIT)), or
|
|
250
|
-
- the Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE)),
|
|
328
|
+
- the MIT license ([LICENSE-MIT](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-MIT)), or
|
|
329
|
+
- the Apache License, Version 2.0 ([LICENSE-APACHE](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-APACHE)),
|
|
251
330
|
|
|
252
331
|
at your option. "At your option" means that whoever uses or redistributes this code picks
|
|
253
332
|
whichever of the two licenses they want to comply with. Nobody has to ask anyone. Unless you say
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Numerical conformance testing for ML code. A Rust core with a Python API and a pytest plugin.
|
|
4
4
|
|
|
5
|
-
`ulpwise` answers
|
|
5
|
+
`ulpwise` answers five questions that come up every time a numerical test goes red on one
|
|
6
6
|
platform and green on another:
|
|
7
7
|
|
|
8
8
|
1. **Which inputs break first?** Named edge values computed from the float format (the largest
|
|
@@ -19,6 +19,10 @@ platform and green on another:
|
|
|
19
19
|
`ulpwise survey` measures 61 elementary and special functions of torch, numpy, scipy and jax in
|
|
20
20
|
ulps against a 200 bit mpmath reference and, for torch, checks every error against the tolerance
|
|
21
21
|
of torch's own `OpInfo` reference test, default and per op override.
|
|
22
|
+
5. **Where should I read first in a repository I do not know?** `ulpwise scan` parses every Python
|
|
23
|
+
file and reports the expressions behind the bugs in the corpus (`exp(x * x)`, `sin(pi * x)`,
|
|
24
|
+
`1 - cos(x)`, `sqrt(a * a + b * b)`, `log(1 + exp(x))`, divisions by an unguarded angle, ...)
|
|
25
|
+
with file, line and function, then lists the functions with the most elementary math.
|
|
22
26
|
|
|
23
27
|
## Why this exists
|
|
24
28
|
|
|
@@ -81,12 +85,15 @@ The Rust crate is usable on its own (`cargo add --git https://github.com/Nichola
|
|
|
81
85
|
```python
|
|
82
86
|
import ulpwise
|
|
83
87
|
|
|
84
|
-
# ulp distances, dtype aware: a float32 tensor is measured in float32 ulps
|
|
88
|
+
# ulp distances, dtype aware: a float32 tensor is measured in float32 ulps, a bfloat16 tensor
|
|
89
|
+
# against a float64 reference in bfloat16 ulps. f64, f32, f16 and bf16.
|
|
85
90
|
ulpwise.ulp_distance(0.9235056042671204, 0.9235056638717651, "f32") # 1
|
|
91
|
+
ulpwise.ulp_distance(1.0, 1.001, "bf16") # 0, both round to 1.0
|
|
86
92
|
ulpwise.assert_max_ulp(torch_out, reference, max_ulp_=2) # floats, lists, numpy, torch
|
|
87
93
|
|
|
88
|
-
# the 29 named edge values of a dtype
|
|
94
|
+
# the 29 named edge values of a dtype, f64 f32 f16 bf16
|
|
89
95
|
dict(ulpwise.special("f32"))["square_underflows_to_zero"] # 2.6469779601696886e-23, the largest x with x * x == 0
|
|
96
|
+
dict(ulpwise.special("bf16"))["square_overflows"] # 1.8446744073709552e+19, 2 ** 64
|
|
90
97
|
|
|
91
98
|
# exact placement of a result relative to its rounding midpoint
|
|
92
99
|
ulpwise.midpoint("sqrt", 0.8528626561164856, "f32") # (0.9235056042671204, 0.000377, True, False)
|
|
@@ -105,8 +112,8 @@ ulpwise.sqrt_cr(0.8528626561164856, "f32") # 0.9235056042671204
|
|
|
105
112
|
|
|
106
113
|
### pytest plugin
|
|
107
114
|
|
|
108
|
-
Installed automatically. A test that takes `edge_f32` or `
|
|
109
|
-
value, and `assert_max_ulp` is available as a fixture:
|
|
115
|
+
Installed automatically. A test that takes `edge_f64`, `edge_f32`, `edge_f16` or `edge_bf16` runs
|
|
116
|
+
once per named edge value, and `assert_max_ulp` is available as a fixture:
|
|
110
117
|
|
|
111
118
|
```python
|
|
112
119
|
def test_my_kernel_survives_the_edges(edge_f32, assert_max_ulp):
|
|
@@ -123,6 +130,9 @@ project grew out of, each with a runnable repro, the buggy behaviour quoted from
|
|
|
123
130
|
request, and the check that tells the two apart. `pytest tests/test_corpus.py` runs every case
|
|
124
131
|
whose packages are installed; a case is an expected failure while the installed release is not
|
|
125
132
|
known to contain the fix, so the run tells you which bugs are present in your environment.
|
|
133
|
+
`ulpwise corpus` does the same without pytest, one line per case with the installed version and
|
|
134
|
+
`present`, `fixed` or `skipped`; `ulpwise corpus --repo pytorch --fail-if-present` is the CI gate
|
|
135
|
+
form.
|
|
126
136
|
|
|
127
137
|
| case | kind | merged |
|
|
128
138
|
|---|---|---|
|
|
@@ -153,6 +163,69 @@ expected failures until a release contains the fix (`fixed_in_release` in `cases
|
|
|
153
163
|
in a temporary directory and needs no weights; two more ultralytics fixes (#26240, #26246) are not in
|
|
154
164
|
the corpus yet because their repros need model weights or the COCO evaluator.
|
|
155
165
|
|
|
166
|
+
## Repository scan
|
|
167
|
+
|
|
168
|
+
```sh
|
|
169
|
+
ulpwise scan kornia/kornia --report kornia.md # clone with depth 1, write a Markdown report
|
|
170
|
+
ulpwise scan . --fail-on high # CI gate: exit 1 on a high severity finding
|
|
171
|
+
ulpwise scan path/to/repo --rules one-minus-cos,small-angle-division --top 40
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
The scan is static and needs nothing installed: it parses each file with `ast`, walks every
|
|
175
|
+
function and matches twelve patterns, each with a severity, the reason it loses digits or
|
|
176
|
+
overflows, the usual replacement and, where one exists, the upstream bug it comes from.
|
|
177
|
+
|
|
178
|
+
| rule | severity | pattern |
|
|
179
|
+
|---|---|---|
|
|
180
|
+
| `exp-of-square` | high | `exp(x * x)`, `exp(x ** 2)`, `(-x.pow(2)).exp()`: the rounding error of the square is multiplied by `x * x / 2` ulps (pytorch #198664) |
|
|
181
|
+
| `sin-of-pi-times` | high | `sin(pi * x)`, `cos(pi * x)`: the product is rounded before the argument reduction (pytorch #198663) |
|
|
182
|
+
| `softplus-by-hand` | high | `log(1 + exp(x))` |
|
|
183
|
+
| `logsumexp-by-hand` | high | `log(exp(a) + exp(b))`, `log(sum(exp(x)))` |
|
|
184
|
+
| `hypot-by-hand` | high | `sqrt(a * a + b * b)` |
|
|
185
|
+
| `sqrt-of-difference` | medium | `sqrt(a - b)` |
|
|
186
|
+
| `one-minus-cos` | medium | `1 - cos(x)` (kornia #4897) |
|
|
187
|
+
| `log1p-by-hand`, `expm1-by-hand` | medium | `log(1 + x)`, `exp(x) - 1` |
|
|
188
|
+
| `atan-of-quotient` | medium | `atan(y / x)` |
|
|
189
|
+
| `small-angle-division` | medium | `/ theta`, `/ theta ** 2`, `/ sin(theta)` in a function that takes `sin` or `cos` of `theta` and has no `where`, `clamp`, `eps` or series in sight (kornia #4838, #4897) |
|
|
190
|
+
| `acos-for-angle` | info | `acos`, `asin` used to recover an angle |
|
|
191
|
+
|
|
192
|
+
On kornia `main` at `e05b0ee` the scan takes 4 s for 506 files and reports 35 findings. The
|
|
193
|
+
`one-minus-cos` and `small-angle-division` findings are the four lines of `So3.right_jacobian` and
|
|
194
|
+
`So3.left_jacobian` that kornia #4897 fixes, `So3.log` (kornia #4838) is under `acos-for-angle`,
|
|
195
|
+
and `Se3.exp` (also #4897) is under `one-minus-cos`. The scan puts `ellipse_to_laf` (kornia #4768)
|
|
196
|
+
on the list too, for a `sqrt` of a difference; the bug there was a different one, so that entry is
|
|
197
|
+
what the scan is: a reading list, not a verdict.
|
|
198
|
+
|
|
199
|
+
`--run` adds the dynamic half. The module level functions with the most elementary math are
|
|
200
|
+
imported and called with the same 91 point grid (both signs of `1e-8` to `1e3`, and zero) for every
|
|
201
|
+
required argument, in float32 and in float64, torch first and numpy second, and the float32 result
|
|
202
|
+
is measured against the float64 one. Two numbers per function: `at scale`, the largest absolute
|
|
203
|
+
error in ulps of the largest output, and `elementwise`, the worst per element ulp distance with
|
|
204
|
+
the input where it happens. Read both. A rotation matrix has entries that should be zero, and
|
|
205
|
+
there the elementwise count compares float32 rounding noise with float64 rounding noise and reaches
|
|
206
|
+
`1e9` while the matrix is fine to one ulp at scale. A function whose output spans forty orders of
|
|
207
|
+
magnitude, a Bessel function, has a meaningless `at scale` number and a meaningful elementwise one.
|
|
208
|
+
A small angle formula without a guard, `(1 - cos(theta)) / theta ** 2`, shows `9e6` in both
|
|
209
|
+
columns. Static methods run; instance methods, functions that need other arguments, fail to import
|
|
210
|
+
or return something that is not a float array are counted with the reason and skipped, never
|
|
211
|
+
guessed at. This imports and runs the repository's code, from the scanned tree, or from the
|
|
212
|
+
package installed in the environment with `--run-installed` when the tree has unbuilt extensions.
|
|
213
|
+
|
|
214
|
+
```sh
|
|
215
|
+
ulpwise scan kornia/kornia --run --run-limit 200 --report kornia.md
|
|
216
|
+
ulpwise scan pytorch/vision --run --run-installed --run-limit 150
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
On kornia main 9 of the 200 busiest functions run as is, the rest are instance methods or want
|
|
220
|
+
shaped inputs. `adjust_log` is on both lists: the static scan flags its `(1 + image).log2()` as
|
|
221
|
+
`log1p-by-hand`, and the run shows `8.7e8` elementwise ulps at `x = 5.6e-8` next to `0.7` ulps at
|
|
222
|
+
scale, which is the right reading for a function defined on images in `[0, 1]`.
|
|
223
|
+
|
|
224
|
+
In ML repositories most `high` findings are learning rate schedules (`cos(pi * progress)` with
|
|
225
|
+
`progress` in `[0, 1]`) and pixel distances (`sqrt(dx * dx + dy * dy)` on coordinates below
|
|
226
|
+
`1e4`), where the argument is bounded and the pattern is harmless. The scan cannot know the bound;
|
|
227
|
+
the reading list is where that judgement happens.
|
|
228
|
+
|
|
156
229
|
## Accuracy survey
|
|
157
230
|
|
|
158
231
|
```sh
|
|
@@ -170,14 +243,18 @@ finite mismatches and the worst input. For torch it also reports how many inputs
|
|
|
170
243
|
kernel and the scalar tail disagree on, and how many inputs would fail torch's reference test under
|
|
171
244
|
the dtype default tolerance and under the op's `OpInfo` override, read from `op_db`.
|
|
172
245
|
|
|
173
|
-
[`studies/accuracy-survey-2026-09`](studies/accuracy-survey-2026-09/README.md) is the first run
|
|
246
|
+
[`studies/accuracy-survey-2026-09`](https://github.com/Nicholas022400701/ulpwise/blob/main/studies/accuracy-survey-2026-09/README.md) is the first run
|
|
174
247
|
(torch 2.14.0+cpu, numpy 2.2.6, scipy 1.18.1, jax 0.11.2, Linux x86_64 AVX512). The short version:
|
|
175
248
|
|
|
176
249
|
- torch's `bessel_j0/j1/y0/y1` and `airy_ai` in float64 are off by 2.6e9 to 3.9e12 ulps and the
|
|
177
|
-
`precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
|
|
250
|
+
`precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
|
|
251
|
+
(pytorch #198583).
|
|
178
252
|
- torch's `polygamma(1, x)` in float64 keeps about 9 digits (4.0e6 ulps, 46 percent of inputs
|
|
179
253
|
beyond 10 ulps) and passes the default float64 tolerance, which at `rtol = atol = 1e-7` tolerates
|
|
180
|
-
about 4.5e8 ulps.
|
|
254
|
+
about 4.5e8 ulps; in float32 it loses every digit for large negative `x` (pytorch #198663).
|
|
255
|
+
- torch's `erfcx` for negative `x` is off by up to `x*x/2` ulps, 44 in float32 at `x = -8.44` and
|
|
256
|
+
157 in float64 at `x = -23.25`, and scipy's float64 `erfcx` returns the same wrong values
|
|
257
|
+
(pytorch #198664).
|
|
181
258
|
- for 12 of 61 torch functions the AVX512 kernel and the scalar tail return different floats for
|
|
182
259
|
the same input, up to 246 of 619 inputs for `mish`.
|
|
183
260
|
- jax on CPU flushes subnormals to zero, its float64 `erfinv` loses 5 digits near the ends of the
|
|
@@ -198,9 +275,11 @@ cross-checks both against `fractions.Fraction` and `decimal.Decimal` at 80 digit
|
|
|
198
275
|
|
|
199
276
|
## Roadmap
|
|
200
277
|
|
|
278
|
+
- `ulpwise scan --run` for methods and functions with tensor shape requirements, by reading the
|
|
279
|
+
docstring and the checks for the shapes, and an mpmath reference for the scalar functions.
|
|
201
280
|
- Mutation scoring for numerical tests: single token mutants of the code under test (`abs`, a
|
|
202
281
|
dropped `sqrt`, `/ 4` for `/ 16`) run against the test suite, reporting which survive.
|
|
203
|
-
- `float16` and `bfloat16`
|
|
282
|
+
- `float16` and `bfloat16` `spacing`, `next_up`, knife edges and exact oracles (ulp distances and edge values are done).
|
|
204
283
|
- Exact references for transcendental functions in Rust (correctly rounded `exp`, `log`, ...) so
|
|
205
284
|
the `f64` knife-edge scans do not need mpmath.
|
|
206
285
|
- Survey backends for CUDA and MPS, and `float16` / `bfloat16` rows.
|
|
@@ -218,8 +297,8 @@ those pull requests carries the same disclosure.
|
|
|
218
297
|
|
|
219
298
|
Licensed under either of
|
|
220
299
|
|
|
221
|
-
- the MIT license ([LICENSE-MIT](LICENSE-MIT)), or
|
|
222
|
-
- the Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE)),
|
|
300
|
+
- the MIT license ([LICENSE-MIT](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-MIT)), or
|
|
301
|
+
- the Apache License, Version 2.0 ([LICENSE-APACHE](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-APACHE)),
|
|
223
302
|
|
|
224
303
|
at your option. "At your option" means that whoever uses or redistributes this code picks
|
|
225
304
|
whichever of the two licenses they want to comply with. Nobody has to ask anyone. Unless you say
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ulpwise"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Numerical conformance testing for ML code: float edge values, exact rounding oracles, knife-edge finders and a pytest plugin"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT OR Apache-2.0" }
|