ulpwise 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {ulpwise-0.2.0 → ulpwise-0.3.0}/.github/workflows/release.yml +19 -1
  2. {ulpwise-0.2.0 → ulpwise-0.3.0}/CHANGELOG.md +31 -0
  3. {ulpwise-0.2.0 → ulpwise-0.3.0}/Cargo.lock +1 -1
  4. {ulpwise-0.2.0 → ulpwise-0.3.0}/Cargo.toml +1 -1
  5. {ulpwise-0.2.0 → ulpwise-0.3.0}/PKG-INFO +91 -12
  6. {ulpwise-0.2.0 → ulpwise-0.3.0}/README.md +90 -11
  7. {ulpwise-0.2.0 → ulpwise-0.3.0}/pyproject.toml +1 -1
  8. ulpwise-0.3.0/python/ulpwise/__init__.py +340 -0
  9. {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/__main__.py +36 -0
  10. {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/corpus/__init__.py +36 -1
  11. {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/corpus/cases.json +2 -1
  12. {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/plugin.py +4 -3
  13. ulpwise-0.3.0/python/ulpwise/scan.py +622 -0
  14. {ulpwise-0.2.0 → ulpwise-0.3.0}/tests/test_core.py +95 -5
  15. ulpwise-0.3.0/tests/test_corpus.py +77 -0
  16. ulpwise-0.3.0/tests/test_scan.py +191 -0
  17. ulpwise-0.2.0/python/ulpwise/__init__.py +0 -184
  18. ulpwise-0.2.0/tests/test_corpus.py +0 -41
  19. {ulpwise-0.2.0 → ulpwise-0.3.0}/.github/workflows/ci.yml +0 -0
  20. {ulpwise-0.2.0 → ulpwise-0.3.0}/.gitignore +0 -0
  21. {ulpwise-0.2.0 → ulpwise-0.3.0}/CONTRIBUTING.md +0 -0
  22. {ulpwise-0.2.0 → ulpwise-0.3.0}/LICENSE-APACHE +0 -0
  23. {ulpwise-0.2.0 → ulpwise-0.3.0}/LICENSE-MIT +0 -0
  24. {ulpwise-0.2.0 → ulpwise-0.3.0}/examples/sqrt_knife_values.py +0 -0
  25. {ulpwise-0.2.0 → ulpwise-0.3.0}/examples/torch_sqrt_conformance.py +0 -0
  26. {ulpwise-0.2.0 → ulpwise-0.3.0}/python/ulpwise/survey.py +0 -0
  27. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/edge.rs +0 -0
  28. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/exact.rs +0 -0
  29. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/knife.rs +0 -0
  30. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/lib.rs +0 -0
  31. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/py.rs +0 -0
  32. {ulpwise-0.2.0 → ulpwise-0.3.0}/src/ulp.rs +0 -0
  33. {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/README.md +0 -0
  34. {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/results.csv +0 -0
  35. {ulpwise-0.2.0 → ulpwise-0.3.0}/studies/accuracy-survey-2026-09/results.md +0 -0
  36. {ulpwise-0.2.0 → ulpwise-0.3.0}/tests/test_survey.py +0 -0
@@ -2,7 +2,8 @@
2
2
  # uploads them to PyPI through trusted publishing (no API token stored anywhere):
3
3
  # the PyPI project "ulpwise" must list this repository, this workflow file name and the
4
4
  # "pypi" environment as its publisher. A manual run only builds and checks the wheels,
5
- # unless its "publish" input is set, in which case it also uploads and tags the commit.
5
+ # unless its "publish" input is set, in which case it also uploads, tags the commit and
6
+ # creates the GitHub release with the CHANGELOG section as notes and the wheels attached.
6
7
  name: release
7
8
 
8
9
  on:
@@ -119,3 +120,20 @@ jobs:
119
120
  run: |
120
121
  git tag "v${{ steps.version.outputs.version }}"
121
122
  git push origin "v${{ steps.version.outputs.version }}"
123
+ - name: GitHub release with the CHANGELOG section as notes and the wheels attached
124
+ env:
125
+ GH_TOKEN: ${{ github.token }}
126
+ VERSION: ${{ steps.version.outputs.version }}
127
+ run: |
128
+ python3 - > notes.md <<'EOF'
129
+ import os, re
130
+ version = os.environ["VERSION"]
131
+ text = open("CHANGELOG.md", encoding="utf-8").read()
132
+ m = re.search(r"^## " + re.escape(version) + r"\b.*?\n(.*?)(?=^## |\Z)", text, re.M | re.S)
133
+ print(m.group(1).strip() if m else f"ulpwise {version}, see CHANGELOG.md")
134
+ EOF
135
+ if gh release view "v$VERSION" > /dev/null 2>&1; then
136
+ gh release upload "v$VERSION" dist/* --clobber
137
+ else
138
+ gh release create "v$VERSION" dist/* --title "ulpwise $VERSION" --notes-file notes.md
139
+ fi
@@ -1,5 +1,36 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0 (2026-09-26)
4
+
5
+ - `ulpwise scan`: static scan of a repository (a directory, a GitHub URL or `owner/repo`, cloned
6
+ with depth 1) for the floating point patterns behind the corpus bugs. Twelve rules with severity,
7
+ reason, replacement and upstream example: `exp-of-square`, `sin-of-pi-times`, `softplus-by-hand`,
8
+ `logsumexp-by-hand`, `hypot-by-hand`, `sqrt-of-difference`, `one-minus-cos`, `log1p-by-hand`,
9
+ `expm1-by-hand`, `atan-of-quotient`, `small-angle-division`, `acos-for-angle`. Findings carry
10
+ file, line, function and the source line; the report ends with the functions that do the most
11
+ elementary math. `--report` writes Markdown, `--rules` filters, `--fail-on` gates CI,
12
+ `--include-tests` widens the walk. On kornia main the four `small-angle-division` and
13
+ `one-minus-cos` lines are the ones kornia #4897 fixes.
14
+ - `ulpwise scan --run`: imports the math-heavy module level functions and calls them on the same
15
+ grid in float32 and float64, reporting the largest error in ulps of the largest output and the
16
+ worst elementwise ulp distance with its input. Static methods run, instance methods and other
17
+ skips carry their reason. `--run-limit` bounds it, `--run-installed` imports the installed package
18
+ instead of the scanned tree.
19
+ - `ulp_distance`, `ulp_distances`, `ordered`, `max_ulp` and `assert_max_ulp` take `f16` and
20
+ `bf16`, and `flatten` reads the dtype off float16 and bfloat16 numpy arrays and torch tensors.
21
+ Inputs that are not representable are rounded to nearest even first, so a bfloat16 tensor can be
22
+ measured against a float64 reference; without an explicit dtype the less precise of the two
23
+ inputs decides. Cross-checked against the numpy int16 view for float16 and the torch view for
24
+ bfloat16.
25
+ - `special("f16")` and `special("bf16")`: the same 29 named edge values as f32 and f64, so
26
+ `edge_values`, the `edge_f16` and `edge_bf16` pytest fixtures and `ulpwise special f16` work.
27
+ Every value satisfies the same checks as the f32 and f64 tables, run through numpy for float16
28
+ and torch for bfloat16.
29
+ - `ulpwise corpus`: runs the regression corpus against the installed packages without pytest and
30
+ prints one line per case, present, fixed or skipped, with the installed version and the upstream
31
+ reference. `--repo` filters by repository or case id, `--fail-if-present` makes a present bug exit 1.
32
+ Any exception from a repro counts as present, like the pytest run: several corpus bugs are crashes.
33
+
3
34
  ## 0.2.0 (2026-09-26)
4
35
 
5
36
  - `ulpwise survey` (`ulpwise.survey`): accuracy survey of 61 elementary and special functions of
@@ -120,7 +120,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
120
120
 
121
121
  [[package]]
122
122
  name = "ulpwise"
123
- version = "0.2.0"
123
+ version = "0.3.0"
124
124
  dependencies = [
125
125
  "pyo3",
126
126
  ]
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "ulpwise"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  edition = "2021"
5
5
  rust-version = "1.86"
6
6
  license = "MIT OR Apache-2.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ulpwise
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Framework :: Pytest
6
6
  Classifier: Intended Audience :: Developers
@@ -30,7 +30,7 @@ Project-URL: Repository, https://github.com/Nicholas022400701/ulpwise
30
30
 
31
31
  Numerical conformance testing for ML code. A Rust core with a Python API and a pytest plugin.
32
32
 
33
- `ulpwise` answers three questions that come up every time a numerical test goes red on one
33
+ `ulpwise` answers five questions that come up every time a numerical test goes red on one
34
34
  platform and green on another:
35
35
 
36
36
  1. **Which inputs break first?** Named edge values computed from the float format (the largest
@@ -47,6 +47,10 @@ platform and green on another:
47
47
  `ulpwise survey` measures 61 elementary and special functions of torch, numpy, scipy and jax in
48
48
  ulps against a 200 bit mpmath reference and, for torch, checks every error against the tolerance
49
49
  of torch's own `OpInfo` reference test, default and per op override.
50
+ 5. **Where should I read first in a repository I do not know?** `ulpwise scan` parses every Python
51
+ file and reports the expressions behind the bugs in the corpus (`exp(x * x)`, `sin(pi * x)`,
52
+ `1 - cos(x)`, `sqrt(a * a + b * b)`, `log(1 + exp(x))`, divisions by an unguarded angle, ...)
53
+ with file, line and function, then lists the functions with the most elementary math.
50
54
 
51
55
  ## Why this exists
52
56
 
@@ -109,12 +113,15 @@ The Rust crate is usable on its own (`cargo add --git https://github.com/Nichola
109
113
  ```python
110
114
  import ulpwise
111
115
 
112
- # ulp distances, dtype aware: a float32 tensor is measured in float32 ulps.
116
+ # ulp distances, dtype aware: a float32 tensor is measured in float32 ulps, a bfloat16 tensor
117
+ # against a float64 reference in bfloat16 ulps. f64, f32, f16 and bf16.
113
118
  ulpwise.ulp_distance(0.9235056042671204, 0.9235056638717651, "f32") # 1
119
+ ulpwise.ulp_distance(1.0, 1.001, "bf16") # 0, both round to 1.0
114
120
  ulpwise.assert_max_ulp(torch_out, reference, max_ulp_=2) # floats, lists, numpy, torch
115
121
 
116
- # the 29 named edge values of a dtype
122
+ # the 29 named edge values of a dtype, f64 f32 f16 bf16
117
123
  dict(ulpwise.special("f32"))["square_underflows_to_zero"] # 2.6469779601696886e-23, the largest x with x * x == 0
124
+ dict(ulpwise.special("bf16"))["square_overflows"] # 1.8446744073709552e+19, 2 ** 64
118
125
 
119
126
  # exact placement of a result relative to its rounding midpoint
120
127
  ulpwise.midpoint("sqrt", 0.8528626561164856, "f32") # (0.9235056042671204, 0.000377, True, False)
@@ -133,8 +140,8 @@ ulpwise.sqrt_cr(0.8528626561164856, "f32") # 0.9235056042671204
133
140
 
134
141
  ### pytest plugin
135
142
 
136
- Installed automatically. A test that takes `edge_f32` or `edge_f64` runs once per named edge
137
- value, and `assert_max_ulp` is available as a fixture:
143
+ Installed automatically. A test that takes `edge_f64`, `edge_f32`, `edge_f16` or `edge_bf16` runs
144
+ once per named edge value, and `assert_max_ulp` is available as a fixture:
138
145
 
139
146
  ```python
140
147
  def test_my_kernel_survives_the_edges(edge_f32, assert_max_ulp):
@@ -151,6 +158,9 @@ project grew out of, each with a runnable repro, the buggy behaviour quoted from
151
158
  request, and the check that tells the two apart. `pytest tests/test_corpus.py` runs every case
152
159
  whose packages are installed; a case is an expected failure while the installed release is not
153
160
  known to contain the fix, so the run tells you which bugs are present in your environment.
161
+ `ulpwise corpus` does the same without pytest, one line per case with the installed version and
162
+ `present`, `fixed` or `skipped`; `ulpwise corpus --repo pytorch --fail-if-present` is the CI gate
163
+ form.
154
164
 
155
165
  | case | kind | merged |
156
166
  |---|---|---|
@@ -181,6 +191,69 @@ expected failures until a release contains the fix (`fixed_in_release` in `cases
181
191
  in a temporary directory and needs no weights; two more ultralytics fixes (#26240, #26246) are not in
182
192
  the corpus yet because their repros need model weights or the COCO evaluator.
183
193
 
194
+ ## Repository scan
195
+
196
+ ```sh
197
+ ulpwise scan kornia/kornia --report kornia.md # clone with depth 1, write a Markdown report
198
+ ulpwise scan . --fail-on high # CI gate: exit 1 on a high severity finding
199
+ ulpwise scan path/to/repo --rules one-minus-cos,small-angle-division --top 40
200
+ ```
201
+
202
+ The scan is static and needs nothing installed: it parses each file with `ast`, walks every
203
+ function and matches twelve patterns, each with a severity, the reason it loses digits or
204
+ overflows, the usual replacement and, where one exists, the upstream bug it comes from.
205
+
206
+ | rule | severity | pattern |
207
+ |---|---|---|
208
+ | `exp-of-square` | high | `exp(x * x)`, `exp(x ** 2)`, `(-x.pow(2)).exp()`: the rounding error of the square is multiplied by `x * x / 2` ulps (pytorch #198664) |
209
+ | `sin-of-pi-times` | high | `sin(pi * x)`, `cos(pi * x)`: the product is rounded before the argument reduction (pytorch #198663) |
210
+ | `softplus-by-hand` | high | `log(1 + exp(x))` |
211
+ | `logsumexp-by-hand` | high | `log(exp(a) + exp(b))`, `log(sum(exp(x)))` |
212
+ | `hypot-by-hand` | high | `sqrt(a * a + b * b)` |
213
+ | `sqrt-of-difference` | medium | `sqrt(a - b)` |
214
+ | `one-minus-cos` | medium | `1 - cos(x)` (kornia #4897) |
215
+ | `log1p-by-hand`, `expm1-by-hand` | medium | `log(1 + x)`, `exp(x) - 1` |
216
+ | `atan-of-quotient` | medium | `atan(y / x)` |
217
+ | `small-angle-division` | medium | `/ theta`, `/ theta ** 2`, `/ sin(theta)` in a function that takes `sin` or `cos` of `theta` and has no `where`, `clamp`, `eps` or series in sight (kornia #4838, #4897) |
218
+ | `acos-for-angle` | info | `acos`, `asin` used to recover an angle |
219
+
220
+ On kornia `main` at `e05b0ee` the scan takes 4 s for 506 files and reports 35 findings. The
221
+ `one-minus-cos` and `small-angle-division` findings are the four lines of `So3.right_jacobian` and
222
+ `So3.left_jacobian` that kornia #4897 fixes, `So3.log` (kornia #4838) is under `acos-for-angle`,
223
+ and `Se3.exp` (also #4897) is under `one-minus-cos`. The scan puts `ellipse_to_laf` (kornia #4768)
224
+ on the list too, for a `sqrt` of a difference; the bug there was a different one, so that entry is
225
+ what the scan is: a reading list, not a verdict.
226
+
227
+ `--run` adds the dynamic half. The module level functions with the most elementary math are
228
+ imported and called with the same 91 point grid (both signs of `1e-8` to `1e3`, and zero) for every
229
+ required argument, in float32 and in float64, torch first and numpy second, and the float32 result
230
+ is measured against the float64 one. Two numbers per function: `at scale`, the largest absolute
231
+ error in ulps of the largest output, and `elementwise`, the worst per element ulp distance with
232
+ the input where it happens. Read both. A rotation matrix has entries that should be zero, and
233
+ there the elementwise count compares float32 rounding noise with float64 rounding noise and reaches
234
+ `1e9` while the matrix is fine to one ulp at scale. A function whose output spans forty orders of
235
+ magnitude, a Bessel function, has a meaningless `at scale` number and a meaningful elementwise one.
236
+ A small angle formula without a guard, `(1 - cos(theta)) / theta ** 2`, shows `9e6` in both
237
+ columns. Static methods run; instance methods, functions that need other arguments, fail to import
238
+ or return something that is not a float array are counted with the reason and skipped, never
239
+ guessed at. This imports and runs the repository's code, from the scanned tree, or from the
240
+ package installed in the environment with `--run-installed` when the tree has unbuilt extensions.
241
+
242
+ ```sh
243
+ ulpwise scan kornia/kornia --run --run-limit 200 --report kornia.md
244
+ ulpwise scan pytorch/vision --run --run-installed --run-limit 150
245
+ ```
246
+
247
+ On kornia main 9 of the 200 busiest functions run as is, the rest are instance methods or want
248
+ shaped inputs. `adjust_log` is on both lists: the static scan flags its `(1 + image).log2()` as
249
+ `log1p-by-hand`, and the run shows `8.7e8` elementwise ulps at `x = 5.6e-8` next to `0.7` ulps at
250
+ scale, which is the right reading for a function defined on images in `[0, 1]`.
251
+
252
+ In ML repositories most `high` findings are learning rate schedules (`cos(pi * progress)` with
253
+ `progress` in `[0, 1]`) and pixel distances (`sqrt(dx * dx + dy * dy)` on coordinates below
254
+ `1e4`), where the argument is bounded and the pattern is harmless. The scan cannot know the bound;
255
+ the reading list is where that judgement happens.
256
+
184
257
  ## Accuracy survey
185
258
 
186
259
  ```sh
@@ -198,14 +271,18 @@ finite mismatches and the worst input. For torch it also reports how many inputs
198
271
  kernel and the scalar tail disagree on, and how many inputs would fail torch's reference test under
199
272
  the dtype default tolerance and under the op's `OpInfo` override, read from `op_db`.
200
273
 
201
- [`studies/accuracy-survey-2026-09`](studies/accuracy-survey-2026-09/README.md) is the first run
274
+ [`studies/accuracy-survey-2026-09`](https://github.com/Nicholas022400701/ulpwise/blob/main/studies/accuracy-survey-2026-09/README.md) is the first run
202
275
  (torch 2.14.0+cpu, numpy 2.2.6, scipy 1.18.1, jax 0.11.2, Linux x86_64 AVX512). The short version:
203
276
 
204
277
  - torch's `bessel_j0/j1/y0/y1` and `airy_ai` in float64 are off by 2.6e9 to 3.9e12 ulps and the
205
- `precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input.
278
+ `precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
279
+ (pytorch #198583).
206
280
  - torch's `polygamma(1, x)` in float64 keeps about 9 digits (4.0e6 ulps, 46 percent of inputs
207
281
  beyond 10 ulps) and passes the default float64 tolerance, which at `rtol = atol = 1e-7` tolerates
208
- about 4.5e8 ulps.
282
+ about 4.5e8 ulps; in float32 it loses every digit for large negative `x` (pytorch #198663).
283
+ - torch's `erfcx` for negative `x` is off by up to `x*x/2` ulps, 44 in float32 at `x = -8.44` and
284
+ 157 in float64 at `x = -23.25`, and scipy's float64 `erfcx` returns the same wrong values
285
+ (pytorch #198664).
209
286
  - for 12 of 61 torch functions the AVX512 kernel and the scalar tail return different floats for
210
287
  the same input, up to 246 of 619 inputs for `mish`.
211
288
  - jax on CPU flushes subnormals to zero, its float64 `erfinv` loses 5 digits near the ends of the
@@ -226,9 +303,11 @@ cross-checks both against `fractions.Fraction` and `decimal.Decimal` at 80 digit
226
303
 
227
304
  ## Roadmap
228
305
 
306
+ - `ulpwise scan --run` for methods and functions with tensor shape requirements, by reading the
307
+ docstring and the checks for the shapes, and an mpmath reference for the scalar functions.
229
308
  - Mutation scoring for numerical tests: single token mutants of the code under test (`abs`, a
230
309
  dropped `sqrt`, `/ 4` for `/ 16`) run against the test suite, reporting which survive.
231
- - `float16` and `bfloat16` ulps and edge values.
310
+ - `float16` and `bfloat16` `spacing`, `next_up`, knife edges and exact oracles (ulp distances and edge values are done).
232
311
  - Exact references for transcendental functions in Rust (correctly rounded `exp`, `log`, ...) so
233
312
  the `f64` knife-edge scans do not need mpmath.
234
313
  - Survey backends for CUDA and MPS, and `float16` / `bfloat16` rows.
@@ -246,8 +325,8 @@ those pull requests carries the same disclosure.
246
325
 
247
326
  Licensed under either of
248
327
 
249
- - the MIT license ([LICENSE-MIT](LICENSE-MIT)), or
250
- - the Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE)),
328
+ - the MIT license ([LICENSE-MIT](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-MIT)), or
329
+ - the Apache License, Version 2.0 ([LICENSE-APACHE](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-APACHE)),
251
330
 
252
331
  at your option. "At your option" means that whoever uses or redistributes this code picks
253
332
  whichever of the two licenses they want to comply with. Nobody has to ask anyone. Unless you say
@@ -2,7 +2,7 @@
2
2
 
3
3
  Numerical conformance testing for ML code. A Rust core with a Python API and a pytest plugin.
4
4
 
5
- `ulpwise` answers three questions that come up every time a numerical test goes red on one
5
+ `ulpwise` answers five questions that come up every time a numerical test goes red on one
6
6
  platform and green on another:
7
7
 
8
8
  1. **Which inputs break first?** Named edge values computed from the float format (the largest
@@ -19,6 +19,10 @@ platform and green on another:
19
19
  `ulpwise survey` measures 61 elementary and special functions of torch, numpy, scipy and jax in
20
20
  ulps against a 200 bit mpmath reference and, for torch, checks every error against the tolerance
21
21
  of torch's own `OpInfo` reference test, default and per op override.
22
+ 5. **Where should I read first in a repository I do not know?** `ulpwise scan` parses every Python
23
+ file and reports the expressions behind the bugs in the corpus (`exp(x * x)`, `sin(pi * x)`,
24
+ `1 - cos(x)`, `sqrt(a * a + b * b)`, `log(1 + exp(x))`, divisions by an unguarded angle, ...)
25
+ with file, line and function, then lists the functions with the most elementary math.
22
26
 
23
27
  ## Why this exists
24
28
 
@@ -81,12 +85,15 @@ The Rust crate is usable on its own (`cargo add --git https://github.com/Nichola
81
85
  ```python
82
86
  import ulpwise
83
87
 
84
- # ulp distances, dtype aware: a float32 tensor is measured in float32 ulps.
88
+ # ulp distances, dtype aware: a float32 tensor is measured in float32 ulps, a bfloat16 tensor
89
+ # against a float64 reference in bfloat16 ulps. f64, f32, f16 and bf16.
85
90
  ulpwise.ulp_distance(0.9235056042671204, 0.9235056638717651, "f32") # 1
91
+ ulpwise.ulp_distance(1.0, 1.001, "bf16") # 0, both round to 1.0
86
92
  ulpwise.assert_max_ulp(torch_out, reference, max_ulp_=2) # floats, lists, numpy, torch
87
93
 
88
- # the 29 named edge values of a dtype
94
+ # the 29 named edge values of a dtype, f64 f32 f16 bf16
89
95
  dict(ulpwise.special("f32"))["square_underflows_to_zero"] # 2.6469779601696886e-23, the largest x with x * x == 0
96
+ dict(ulpwise.special("bf16"))["square_overflows"] # 1.8446744073709552e+19, 2 ** 64
90
97
 
91
98
  # exact placement of a result relative to its rounding midpoint
92
99
  ulpwise.midpoint("sqrt", 0.8528626561164856, "f32") # (0.9235056042671204, 0.000377, True, False)
@@ -105,8 +112,8 @@ ulpwise.sqrt_cr(0.8528626561164856, "f32") # 0.9235056042671204
105
112
 
106
113
  ### pytest plugin
107
114
 
108
- Installed automatically. A test that takes `edge_f32` or `edge_f64` runs once per named edge
109
- value, and `assert_max_ulp` is available as a fixture:
115
+ Installed automatically. A test that takes `edge_f64`, `edge_f32`, `edge_f16` or `edge_bf16` runs
116
+ once per named edge value, and `assert_max_ulp` is available as a fixture:
110
117
 
111
118
  ```python
112
119
  def test_my_kernel_survives_the_edges(edge_f32, assert_max_ulp):
@@ -123,6 +130,9 @@ project grew out of, each with a runnable repro, the buggy behaviour quoted from
123
130
  request, and the check that tells the two apart. `pytest tests/test_corpus.py` runs every case
124
131
  whose packages are installed; a case is an expected failure while the installed release is not
125
132
  known to contain the fix, so the run tells you which bugs are present in your environment.
133
+ `ulpwise corpus` does the same without pytest, one line per case with the installed version and
134
+ `present`, `fixed` or `skipped`; `ulpwise corpus --repo pytorch --fail-if-present` is the CI gate
135
+ form.
126
136
 
127
137
  | case | kind | merged |
128
138
  |---|---|---|
@@ -153,6 +163,69 @@ expected failures until a release contains the fix (`fixed_in_release` in `cases
153
163
  in a temporary directory and needs no weights; two more ultralytics fixes (#26240, #26246) are not in
154
164
  the corpus yet because their repros need model weights or the COCO evaluator.
155
165
 
166
+ ## Repository scan
167
+
168
+ ```sh
169
+ ulpwise scan kornia/kornia --report kornia.md # clone with depth 1, write a Markdown report
170
+ ulpwise scan . --fail-on high # CI gate: exit 1 on a high severity finding
171
+ ulpwise scan path/to/repo --rules one-minus-cos,small-angle-division --top 40
172
+ ```
173
+
174
+ The scan is static and needs nothing installed: it parses each file with `ast`, walks every
175
+ function and matches twelve patterns, each with a severity, the reason it loses digits or
176
+ overflows, the usual replacement and, where one exists, the upstream bug it comes from.
177
+
178
+ | rule | severity | pattern |
179
+ |---|---|---|
180
+ | `exp-of-square` | high | `exp(x * x)`, `exp(x ** 2)`, `(-x.pow(2)).exp()`: the rounding error of the square is multiplied by `x * x / 2` ulps (pytorch #198664) |
181
+ | `sin-of-pi-times` | high | `sin(pi * x)`, `cos(pi * x)`: the product is rounded before the argument reduction (pytorch #198663) |
182
+ | `softplus-by-hand` | high | `log(1 + exp(x))` |
183
+ | `logsumexp-by-hand` | high | `log(exp(a) + exp(b))`, `log(sum(exp(x)))` |
184
+ | `hypot-by-hand` | high | `sqrt(a * a + b * b)` |
185
+ | `sqrt-of-difference` | medium | `sqrt(a - b)` |
186
+ | `one-minus-cos` | medium | `1 - cos(x)` (kornia #4897) |
187
+ | `log1p-by-hand`, `expm1-by-hand` | medium | `log(1 + x)`, `exp(x) - 1` |
188
+ | `atan-of-quotient` | medium | `atan(y / x)` |
189
+ | `small-angle-division` | medium | `/ theta`, `/ theta ** 2`, `/ sin(theta)` in a function that takes `sin` or `cos` of `theta` and has no `where`, `clamp`, `eps` or series in sight (kornia #4838, #4897) |
190
+ | `acos-for-angle` | info | `acos`, `asin` used to recover an angle |
191
+
192
+ On kornia `main` at `e05b0ee` the scan takes 4 s for 506 files and reports 35 findings. The
193
+ `one-minus-cos` and `small-angle-division` findings are the four lines of `So3.right_jacobian` and
194
+ `So3.left_jacobian` that kornia #4897 fixes, `So3.log` (kornia #4838) is under `acos-for-angle`,
195
+ and `Se3.exp` (also #4897) is under `one-minus-cos`. The scan puts `ellipse_to_laf` (kornia #4768)
196
+ on the list too, for a `sqrt` of a difference; the bug there was a different one, so that entry is
197
+ what the scan is: a reading list, not a verdict.
198
+
199
+ `--run` adds the dynamic half. The module level functions with the most elementary math are
200
+ imported and called with the same 91 point grid (both signs of `1e-8` to `1e3`, and zero) for every
201
+ required argument, in float32 and in float64, torch first and numpy second, and the float32 result
202
+ is measured against the float64 one. Two numbers per function: `at scale`, the largest absolute
203
+ error in ulps of the largest output, and `elementwise`, the worst per element ulp distance with
204
+ the input where it happens. Read both. A rotation matrix has entries that should be zero, and
205
+ there the elementwise count compares float32 rounding noise with float64 rounding noise and reaches
206
+ `1e9` while the matrix is fine to one ulp at scale. A function whose output spans forty orders of
207
+ magnitude, a Bessel function, has a meaningless `at scale` number and a meaningful elementwise one.
208
+ A small angle formula without a guard, `(1 - cos(theta)) / theta ** 2`, shows `9e6` in both
209
+ columns. Static methods run; instance methods, functions that need other arguments, fail to import
210
+ or return something that is not a float array are counted with the reason and skipped, never
211
+ guessed at. This imports and runs the repository's code, from the scanned tree, or from the
212
+ package installed in the environment with `--run-installed` when the tree has unbuilt extensions.
213
+
214
+ ```sh
215
+ ulpwise scan kornia/kornia --run --run-limit 200 --report kornia.md
216
+ ulpwise scan pytorch/vision --run --run-installed --run-limit 150
217
+ ```
218
+
219
+ On kornia main 9 of the 200 busiest functions run as is, the rest are instance methods or want
220
+ shaped inputs. `adjust_log` is on both lists: the static scan flags its `(1 + image).log2()` as
221
+ `log1p-by-hand`, and the run shows `8.7e8` elementwise ulps at `x = 5.6e-8` next to `0.7` ulps at
222
+ scale, which is the right reading for a function defined on images in `[0, 1]`.
223
+
224
+ In ML repositories most `high` findings are learning rate schedules (`cos(pi * progress)` with
225
+ `progress` in `[0, 1]`) and pixel distances (`sqrt(dx * dx + dy * dy)` on coordinates below
226
+ `1e4`), where the argument is bounded and the pattern is harmless. The scan cannot know the bound;
227
+ the reading list is where that judgement happens.
228
+
156
229
  ## Accuracy survey
157
230
 
158
231
  ```sh
@@ -170,14 +243,18 @@ finite mismatches and the worst input. For torch it also reports how many inputs
170
243
  kernel and the scalar tail disagree on, and how many inputs would fail torch's reference test under
171
244
  the dtype default tolerance and under the op's `OpInfo` override, read from `op_db`.
172
245
 
173
- [`studies/accuracy-survey-2026-09`](studies/accuracy-survey-2026-09/README.md) is the first run
246
+ [`studies/accuracy-survey-2026-09`](https://github.com/Nicholas022400701/ulpwise/blob/main/studies/accuracy-survey-2026-09/README.md) is the first run
174
247
  (torch 2.14.0+cpu, numpy 2.2.6, scipy 1.18.1, jax 0.11.2, Linux x86_64 AVX512). The short version:
175
248
 
176
249
  - torch's `bessel_j0/j1/y0/y1` and `airy_ai` in float64 are off by 2.6e9 to 3.9e12 ulps and the
177
- `precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input.
250
+ `precisionOverride({torch.float64: 1e-05})` on their tests hides every failing input
251
+ (pytorch #198583).
178
252
  - torch's `polygamma(1, x)` in float64 keeps about 9 digits (4.0e6 ulps, 46 percent of inputs
179
253
  beyond 10 ulps) and passes the default float64 tolerance, which at `rtol = atol = 1e-7` tolerates
180
- about 4.5e8 ulps.
254
+ about 4.5e8 ulps; in float32 it loses every digit for large negative `x` (pytorch #198663).
255
+ - torch's `erfcx` for negative `x` is off by up to `x*x/2` ulps, 44 in float32 at `x = -8.44` and
256
+ 157 in float64 at `x = -23.25`, and scipy's float64 `erfcx` returns the same wrong values
257
+ (pytorch #198664).
181
258
  - for 12 of 61 torch functions the AVX512 kernel and the scalar tail return different floats for
182
259
  the same input, up to 246 of 619 inputs for `mish`.
183
260
  - jax on CPU flushes subnormals to zero, its float64 `erfinv` loses 5 digits near the ends of the
@@ -198,9 +275,11 @@ cross-checks both against `fractions.Fraction` and `decimal.Decimal` at 80 digit
198
275
 
199
276
  ## Roadmap
200
277
 
278
+ - `ulpwise scan --run` for methods and functions with tensor shape requirements, by reading the
279
+ docstring and the checks for the shapes, and an mpmath reference for the scalar functions.
201
280
  - Mutation scoring for numerical tests: single token mutants of the code under test (`abs`, a
202
281
  dropped `sqrt`, `/ 4` for `/ 16`) run against the test suite, reporting which survive.
203
- - `float16` and `bfloat16` ulps and edge values.
282
+ - `float16` and `bfloat16` `spacing`, `next_up`, knife edges and exact oracles (ulp distances and edge values are done).
204
283
  - Exact references for transcendental functions in Rust (correctly rounded `exp`, `log`, ...) so
205
284
  the `f64` knife-edge scans do not need mpmath.
206
285
  - Survey backends for CUDA and MPS, and `float16` / `bfloat16` rows.
@@ -218,8 +297,8 @@ those pull requests carries the same disclosure.
218
297
 
219
298
  Licensed under either of
220
299
 
221
- - the MIT license ([LICENSE-MIT](LICENSE-MIT)), or
222
- - the Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE)),
300
+ - the MIT license ([LICENSE-MIT](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-MIT)), or
301
+ - the Apache License, Version 2.0 ([LICENSE-APACHE](https://github.com/Nicholas022400701/ulpwise/blob/main/LICENSE-APACHE)),
223
302
 
224
303
  at your option. "At your option" means that whoever uses or redistributes this code picks
225
304
  whichever of the two licenses they want to comply with. Nobody has to ask anyone. Unless you say
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "ulpwise"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "Numerical conformance testing for ML code: float edge values, exact rounding oracles, knife-edge finders and a pytest plugin"
9
9
  readme = "README.md"
10
10
  license = { text = "MIT OR Apache-2.0" }