obelisk-env 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- obelisk_env-0.1.0/LICENSE +21 -0
- obelisk_env-0.1.0/PKG-INFO +324 -0
- obelisk_env-0.1.0/README.md +302 -0
- obelisk_env-0.1.0/obelisk/__init__.py +12 -0
- obelisk_env-0.1.0/obelisk/backend.py +136 -0
- obelisk_env-0.1.0/obelisk/check.py +243 -0
- obelisk_env-0.1.0/obelisk/cli.py +680 -0
- obelisk_env-0.1.0/obelisk/compare.py +269 -0
- obelisk_env-0.1.0/obelisk/coverage.py +297 -0
- obelisk_env-0.1.0/obelisk/fingerprint.py +235 -0
- obelisk_env-0.1.0/obelisk/inspect.py +152 -0
- obelisk_env-0.1.0/obelisk/migrate.py +146 -0
- obelisk_env-0.1.0/obelisk/obl.py +75 -0
- obelisk_env-0.1.0/obelisk/probes.py +370 -0
- obelisk_env-0.1.0/obelisk/profile.py +129 -0
- obelisk_env-0.1.0/obelisk/report.py +380 -0
- obelisk_env-0.1.0/obelisk/shim.py +498 -0
- obelisk_env-0.1.0/obelisk/workload.py +137 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/PKG-INFO +324 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/SOURCES.txt +24 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/dependency_links.txt +1 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/entry_points.txt +2 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/requires.txt +3 -0
- obelisk_env-0.1.0/obelisk_env.egg-info/top_level.txt +1 -0
- obelisk_env-0.1.0/pyproject.toml +37 -0
- obelisk_env-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Beijing Huahuo Zhichuang Technology Co., Ltd.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: obelisk-env
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Verify that your code runs correctly on a different accelerator, not just that it runs.
|
|
5
|
+
Author: Beijing Huahuo Zhichuang Technology Co., Ltd.
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: accelerator,gpu,npu,cuda,musa,ascend,pytorch,porting,migration,numerical,verification,conformance,reproducibility,hardware,domestic-ai-chips
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Classifier: Topic :: System :: Hardware
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: torch
|
|
20
|
+
Requires-Dist: torch; extra == "torch"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# Obelisk
|
|
24
|
+
|
|
25
|
+
**Your code runs on the new accelerator. That is not the same as running correctly.**
|
|
26
|
+
|
|
27
|
+
Moving a model from one accelerator to another is usually declared finished when
|
|
28
|
+
the program stops crashing. Nothing checks the part that matters: whether the
|
|
29
|
+
numbers coming out are still the numbers that came out before. There is no
|
|
30
|
+
baseline to check them against, so "it ran" becomes the whole verdict.
|
|
31
|
+
|
|
32
|
+
Obelisk supplies the missing baseline. It measures a reference device, writes the
|
|
33
|
+
measurements to a small file, carries that file to the target device, and reports
|
|
34
|
+
every place the two disagree, how far apart they are, and which hardware
|
|
35
|
+
assumption is responsible.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
pip install obelisk-ai
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## The shortest version
|
|
42
|
+
|
|
43
|
+
Run the same two lines on two machines.
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
$ obe sample # on the NVIDIA box $ obe sample # on the domestic card
|
|
47
|
+
|
|
48
|
+
device NVIDIA GeForce RTX 4090 device MTT S4000
|
|
49
|
+
torch 2.2.0+cu121 torch 2.2.0
|
|
50
|
+
|
|
51
|
+
rand (uniform) +0.39904648 +0.51667917 rand (uniform) +0.39904648 +0.51667917
|
|
52
|
+
randn (normal) -0.92466241 -0.42534423 randn (normal) -0.92466241 -0.42534396
|
|
53
|
+
log1p +0.69314718 +1.09861231 log1p NotImplementedError
|
|
54
|
+
digamma -0.57721591 +0.42278421 digamma NotImplementedError
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Same seed, same code, same version of PyTorch. The uniform random stream matches
|
|
58
|
+
bit for bit. The normal one does not. Three library functions are missing
|
|
59
|
+
outright. Nothing here raised a warning on the way to that difference, and
|
|
60
|
+
nothing in either program would have told you.
|
|
61
|
+
|
|
62
|
+
## The full version
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
# on the reference machine
|
|
66
|
+
obe record --out reference.obl
|
|
67
|
+
|
|
68
|
+
# copy reference.obl to the target machine — it is a small text file and it is
|
|
69
|
+
# everything the target needs
|
|
70
|
+
|
|
71
|
+
# on the target machine
|
|
72
|
+
obe verify --ref reference.obl --html report.html
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
ENVIRONMENT ALIGNMENT REPORT GRADE D
|
|
77
|
+
|
|
78
|
+
reference NVIDIA GeForce RTX 4090 (cuda, torch 2.2.0+cu121)
|
|
79
|
+
target MTT S4000 (musa, torch 2.2.0)
|
|
80
|
+
|
|
81
|
+
ASSUMPTIONS
|
|
82
|
+
x lib.transcendental Transcendental functions ERROR aten::log1p is missing
|
|
83
|
+
! acc.fp16_accumulate Half-precision accumulator DIVERGENT 3.37e-04
|
|
84
|
+
! ord.atomics Atomic accumulation order DIVERGENT 2.40e-03
|
|
85
|
+
! lib.normalisation Normalisation epsilon DIVERGENT 1.32e-05
|
|
86
|
+
! rng.device_stream Device random stream NOT_BITWISE
|
|
87
|
+
! sel.tie_breaking Tie breaking in sort/topk NOT_BITWISE
|
|
88
|
+
~ acc.fp32_matmul float32 matmul precision WITHIN_TOLERANCE
|
|
89
|
+
error_vs_exact reference 2.93e-05 target 9.49e-05
|
|
90
|
+
mantissa_bits reference 21.7 target 20.0
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The last two lines are a measurement rather than an accusation. Both devices are
|
|
94
|
+
compared against the same product computed exactly in float64 on the CPU, a
|
|
95
|
+
yardstick that belongs to neither of them, so each one's precision can be stated
|
|
96
|
+
on its own terms.
|
|
97
|
+
|
|
98
|
+
## The whole device, not just the fifteen questions
|
|
99
|
+
|
|
100
|
+
`obe record` measures the assumptions a workload depends on. `obe dump` measures
|
|
101
|
+
the device itself: every operator PyTorch exposes, every data type, and the
|
|
102
|
+
probes. The method rests on one decision. The CPU is the oracle for whether a
|
|
103
|
+
call is *valid*, and the device is the subject. Every case runs on the CPU first;
|
|
104
|
+
if it fails there, the harness called it wrong and the case is discarded rather
|
|
105
|
+
than counted against the device. Only a call the CPU accepts and the device
|
|
106
|
+
refuses is a finding.
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
obe dump --out card.obl # on each card, once
|
|
110
|
+
obe diff a.obl b.obl # anywhere, with no hardware present
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Measured on an RTX 4090 against a Moore Threads MTT S4000, both on torch 2.2.0.
|
|
114
|
+
Seven hundred callables discovered, 399 valid on the CPU and run on both:
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
OPERATOR COVERAGE (399 operators run on both)
|
|
118
|
+
behave the same 272
|
|
119
|
+
numerically different 20
|
|
120
|
+
unavailable on the target 105 (73 distinct operations)
|
|
121
|
+
|
|
122
|
+
WORKS ON THE REFERENCE, NOT ON THE TARGET
|
|
123
|
+
x std median svd qr cholesky det logdet
|
|
124
|
+
x asin asinh acosh atanh sinh cosh log1p
|
|
125
|
+
x lgamma digamma erfc exp2 hypot nansum kthvalue
|
|
126
|
+
and 52 more
|
|
127
|
+
|
|
128
|
+
RUNS ON BOTH, RETURNS A DIFFERENT ANSWER
|
|
129
|
+
! torch.selu MISMATCH 4.83e-02
|
|
130
|
+
! torch.vander MISMATCH 1.00e+00
|
|
131
|
+
! torch.max_pool1d_with_indices SHAPE
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
SELU multiplies positive inputs by 1.0507. On that card it returns them
|
|
135
|
+
unchanged, so every positive activation is five percent low, the negative branch
|
|
136
|
+
is correct, and nothing raises. A model built on it trains, converges, and is
|
|
137
|
+
wrong. That is the case no amount of running the program would have found.
|
|
138
|
+
|
|
139
|
+
Each profile carries a digest over everything it asserts, and a comparison cites
|
|
140
|
+
the digests it was built from, so a report can be checked against the
|
|
141
|
+
measurements it came from and a withdrawn measurement can be named exactly.
|
|
142
|
+
|
|
143
|
+
Profiles accumulate. Ten cards is ten hardware sessions and forty-five pairings,
|
|
144
|
+
and every pairing after the first is a file read.
|
|
145
|
+
|
|
146
|
+
## Making it run, and saying what that cost
|
|
147
|
+
|
|
148
|
+
A missing operator is rarely a missing capability. log1p is log(1+x), asinh is a
|
|
149
|
+
logarithm and a square root, a standard deviation is the square root of a
|
|
150
|
+
variance. The hardware can nearly always do the work; what it lacks is someone to
|
|
151
|
+
write the arrangement down.
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
obe run train.py
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
patches what the device cannot do and runs the program. Three kinds of
|
|
158
|
+
substitution, and every one of them recorded:
|
|
159
|
+
|
|
160
|
+
| | |
|
|
161
|
+
|---|---|
|
|
162
|
+
| rewrite | expressed in operators the device does have, still on the device |
|
|
163
|
+
| fallback | computed on the CPU, which is correct and slow, and the ledger says how slow |
|
|
164
|
+
| correct | the device has the operator, and the operator is wrong |
|
|
165
|
+
|
|
166
|
+
On the MTT S4000 an ordinary piece of scientific PyTorch dies at line 11 with
|
|
167
|
+
`aten::std.correction`. Under `obe run` it completes, having stood in for 66
|
|
168
|
+
operators, and finishes with this:
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
LEDGER: what did not happen the way you wrote it
|
|
172
|
+
|
|
173
|
+
operator how calls seconds
|
|
174
|
+
selu correct 2 0.000
|
|
175
|
+
slogdet fallback 1 0.001
|
|
176
|
+
std rewrite 1 0.000
|
|
177
|
+
median rewrite 1 0.054
|
|
178
|
+
...
|
|
179
|
+
10 call(s) through 9 substituted operator(s), 0.056 s total
|
|
180
|
+
0.001 s of that was spent moving data to the CPU and back
|
|
181
|
+
selu was called 2 time(s); the device kernel is wrong, so the result you
|
|
182
|
+
got is not the result the device would have given
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The point is not that the program stopped crashing. Anything can stop a program
|
|
186
|
+
crashing. The point is the last line, which a silent compatibility layer would
|
|
187
|
+
never tell you.
|
|
188
|
+
|
|
189
|
+
## When the card and the code are in different places
|
|
190
|
+
|
|
191
|
+
They almost always are. The code is on a laptop or a build server; the card is in
|
|
192
|
+
a machine room with no route out, and nobody has put the code on it yet. That is
|
|
193
|
+
exactly why nobody checks.
|
|
194
|
+
|
|
195
|
+
```
|
|
196
|
+
obe dump --out s4000.obl # on the card, with none of your code present
|
|
197
|
+
scp s4000.obl laptop: # one file, under a megabyte
|
|
198
|
+
obe check ./repo --card s4000.obl # on a box with no accelerator and no torch
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
```
|
|
202
|
+
CODE AGAINST CARD GRADE C
|
|
203
|
+
|
|
204
|
+
code ./normalizing-flows (71 python files)
|
|
205
|
+
card MTT S4000 (musa)
|
|
206
|
+
|
|
207
|
+
7 operations this code calls would fail here, and all of them can be substituted.
|
|
208
|
+
|
|
209
|
+
WILL NOT RUN
|
|
210
|
+
x torch.slogdet MISSING obe can run it on the CPU 4 call sites
|
|
211
|
+
normflows/flows/mixing.py:117
|
|
212
|
+
x torch.std MISSING obe can rewrite it 3 call sites
|
|
213
|
+
normflows/flows/normalization.py:23
|
|
214
|
+
x F.logsigmoid MISSING obe can rewrite it 2 call sites
|
|
215
|
+
x torch.qr MISSING obe can run it on the CPU 2 call sites
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Nobody bought anything, nobody shipped code into a machine room, and nobody
|
|
219
|
+
waited for a slot on a card.
|
|
220
|
+
|
|
221
|
+
## What it checks
|
|
222
|
+
|
|
223
|
+
Fifteen probes, each isolating one assumption that programs make about their
|
|
224
|
+
accelerator and never write down.
|
|
225
|
+
|
|
226
|
+
| group | assumption |
|
|
227
|
+
|---|---|
|
|
228
|
+
| accumulation | summation order, half-precision accumulator width |
|
|
229
|
+
| precision | float32 matmul precision, bfloat16 support |
|
|
230
|
+
| math library | transcendental functions, softmax numerics, normalisation epsilon |
|
|
231
|
+
| parallel ordering | atomic accumulation order, run-to-run stability, reduction over awkward shapes |
|
|
232
|
+
| randomness | the device random stream |
|
|
233
|
+
| selection | tie breaking in sort, topk and argmax |
|
|
234
|
+
| fused kernels | scaled dot-product attention, convolution |
|
|
235
|
+
| memory | layout and striding |
|
|
236
|
+
|
|
237
|
+
Then the same questions are asked of a real transformer: every layer of a forward
|
|
238
|
+
pass, and the loss and parameters across fifty training steps.
|
|
239
|
+
|
|
240
|
+
## How a difference becomes an explanation
|
|
241
|
+
|
|
242
|
+
A layer that disagrees is a symptom. The probe that disagrees in the same way is
|
|
243
|
+
the cause. When a layer diverges, Obelisk names the probes that could explain it
|
|
244
|
+
and that are themselves failing, so the report ends with sentences like *this
|
|
245
|
+
normalisation layer diverges, and the normalisation epsilon probe is also
|
|
246
|
+
diverging* rather than a wall of numbers.
|
|
247
|
+
|
|
248
|
+
## Grades
|
|
249
|
+
|
|
250
|
+
| | |
|
|
251
|
+
|---|---|
|
|
252
|
+
| **A** | bit-identical throughout |
|
|
253
|
+
| **B** | differs, but every difference is inside the tolerance for its dtype |
|
|
254
|
+
| **C** | at least one assumption is broken; usable with care |
|
|
255
|
+
| **D** | a critical assumption is broken, or an operation is missing |
|
|
256
|
+
| **E** | the device does not agree with itself |
|
|
257
|
+
|
|
258
|
+
## Commands
|
|
259
|
+
|
|
260
|
+
| | |
|
|
261
|
+
|---|---|
|
|
262
|
+
| `obe sample` | print a few numbers; run it on two machines and read them side by side |
|
|
263
|
+
| `obe run` | run a program with the gaps filled in, and report every substitution |
|
|
264
|
+
| `obe check` | judge a repository against a card profiled somewhere else |
|
|
265
|
+
| `obe dump` | profile this device completely: operators, dtypes, probes |
|
|
266
|
+
| `obe diff a.obl b.obl` | compare two profiles with no hardware present |
|
|
267
|
+
| `obe record` | measure the reference for one workload |
|
|
268
|
+
| `obe verify --ref` | compare this device against a recorded reference |
|
|
269
|
+
| `obe inspect` | read a repository and predict its risk, before any card exists |
|
|
270
|
+
| `obe migrate` | rewrite for another backend, and report what it could not change |
|
|
271
|
+
| `obe probes` | list the assumptions Obelisk knows how to test |
|
|
272
|
+
| `obe devices` | list the accelerators visible here |
|
|
273
|
+
|
|
274
|
+
A grade is never the whole answer, and the report says what drove it.
|
|
275
|
+
|
|
276
|
+
## Before you have a card
|
|
277
|
+
|
|
278
|
+
```
|
|
279
|
+
obe inspect ./your-repo
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
reads the source and predicts the risk from the assumptions written into it:
|
|
283
|
+
device names, pinned backends, hand-written kernels, precision flags,
|
|
284
|
+
order-dependent accumulations. On nanoGPT it finds 81 of them and returns a
|
|
285
|
+
predicted grade of C. The prediction is written down before the run, so it can
|
|
286
|
+
be held against the result afterwards.
|
|
287
|
+
|
|
288
|
+
```
|
|
289
|
+
obe migrate ./your-repo --target musa --patch port.diff
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
rewrites the mechanical part and, more usefully, reports what it could not safely
|
|
293
|
+
change, including every place the old backend is still named after the rewrite.
|
|
294
|
+
|
|
295
|
+
## Why the blueprint is a text file
|
|
296
|
+
|
|
297
|
+
Reference and target are usually not the same machine, and the interesting one
|
|
298
|
+
usually has no network. A blueprint is about a megabyte of JSON. It travels on a
|
|
299
|
+
USB stick, it can be opened and read by a person, and it carries enough of each
|
|
300
|
+
tensor — a digest, summary statistics, and a deterministic sample of the values —
|
|
301
|
+
that the two sides can be compared elementwise without either tensor ever being
|
|
302
|
+
in the same room as the other.
|
|
303
|
+
|
|
304
|
+
Nothing of your code or your data goes into it. Obelisk measures the device, not
|
|
305
|
+
the workload you are protecting.
|
|
306
|
+
|
|
307
|
+
## Tolerance
|
|
308
|
+
|
|
309
|
+
The headline number for every comparison is the largest absolute disagreement
|
|
310
|
+
divided by the size of the largest value in the reference. Relative error on its
|
|
311
|
+
own is unusable here: two values of 1e-41 that differ by 1e-41 are a hundred
|
|
312
|
+
percent apart and both of them are zero.
|
|
313
|
+
|
|
314
|
+
Quantities that have no excuse to differ at all — a random stream, the order of a
|
|
315
|
+
sort — are held to a stricter standard and must match exactly.
|
|
316
|
+
|
|
317
|
+
## Supported backends
|
|
318
|
+
|
|
319
|
+
CUDA, MUSA (Moore Threads), and CPU. A backend is three small methods; adding one
|
|
320
|
+
does not touch anything else.
|
|
321
|
+
|
|
322
|
+
## License
|
|
323
|
+
|
|
324
|
+
MIT.
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
# Obelisk
|
|
2
|
+
|
|
3
|
+
**Your code runs on the new accelerator. That is not the same as running correctly.**
|
|
4
|
+
|
|
5
|
+
Moving a model from one accelerator to another is usually declared finished when
|
|
6
|
+
the program stops crashing. Nothing checks the part that matters: whether the
|
|
7
|
+
numbers coming out are still the numbers that came out before. There is no
|
|
8
|
+
baseline to check them against, so "it ran" becomes the whole verdict.
|
|
9
|
+
|
|
10
|
+
Obelisk supplies the missing baseline. It measures a reference device, writes the
|
|
11
|
+
measurements to a small file, carries that file to the target device, and reports
|
|
12
|
+
every place the two disagree, how far apart they are, and which hardware
|
|
13
|
+
assumption is responsible.
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
pip install obelisk-ai
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## The shortest version
|
|
20
|
+
|
|
21
|
+
Run the same two lines on two machines.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
$ obe sample # on the NVIDIA box $ obe sample # on the domestic card
|
|
25
|
+
|
|
26
|
+
device NVIDIA GeForce RTX 4090 device MTT S4000
|
|
27
|
+
torch 2.2.0+cu121 torch 2.2.0
|
|
28
|
+
|
|
29
|
+
rand (uniform) +0.39904648 +0.51667917 rand (uniform) +0.39904648 +0.51667917
|
|
30
|
+
randn (normal) -0.92466241 -0.42534423 randn (normal) -0.92466241 -0.42534396
|
|
31
|
+
log1p +0.69314718 +1.09861231 log1p NotImplementedError
|
|
32
|
+
digamma -0.57721591 +0.42278421 digamma NotImplementedError
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Same seed, same code, same version of PyTorch. The uniform random stream matches
|
|
36
|
+
bit for bit. The normal one does not. Three library functions are missing
|
|
37
|
+
outright. Nothing here raised a warning on the way to that difference, and
|
|
38
|
+
nothing in either program would have told you.
|
|
39
|
+
|
|
40
|
+
## The full version
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
# on the reference machine
|
|
44
|
+
obe record --out reference.obl
|
|
45
|
+
|
|
46
|
+
# copy reference.obl to the target machine — it is a small text file and it is
|
|
47
|
+
# everything the target needs
|
|
48
|
+
|
|
49
|
+
# on the target machine
|
|
50
|
+
obe verify --ref reference.obl --html report.html
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
ENVIRONMENT ALIGNMENT REPORT GRADE D
|
|
55
|
+
|
|
56
|
+
reference NVIDIA GeForce RTX 4090 (cuda, torch 2.2.0+cu121)
|
|
57
|
+
target MTT S4000 (musa, torch 2.2.0)
|
|
58
|
+
|
|
59
|
+
ASSUMPTIONS
|
|
60
|
+
x lib.transcendental Transcendental functions ERROR aten::log1p is missing
|
|
61
|
+
! acc.fp16_accumulate Half-precision accumulator DIVERGENT 3.37e-04
|
|
62
|
+
! ord.atomics Atomic accumulation order DIVERGENT 2.40e-03
|
|
63
|
+
! lib.normalisation Normalisation epsilon DIVERGENT 1.32e-05
|
|
64
|
+
! rng.device_stream Device random stream NOT_BITWISE
|
|
65
|
+
! sel.tie_breaking Tie breaking in sort/topk NOT_BITWISE
|
|
66
|
+
~ acc.fp32_matmul float32 matmul precision WITHIN_TOLERANCE
|
|
67
|
+
error_vs_exact reference 2.93e-05 target 9.49e-05
|
|
68
|
+
mantissa_bits reference 21.7 target 20.0
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The last two lines are a measurement rather than an accusation. Both devices are
|
|
72
|
+
compared against the same product computed exactly in float64 on the CPU, a
|
|
73
|
+
yardstick that belongs to neither of them, so each one's precision can be stated
|
|
74
|
+
on its own terms.
|
|
75
|
+
|
|
76
|
+
## The whole device, not just the fifteen questions
|
|
77
|
+
|
|
78
|
+
`obe record` measures the assumptions a workload depends on. `obe dump` measures
|
|
79
|
+
the device itself: every operator PyTorch exposes, every data type, and the
|
|
80
|
+
probes. The method rests on one decision. The CPU is the oracle for whether a
|
|
81
|
+
call is *valid*, and the device is the subject. Every case runs on the CPU first;
|
|
82
|
+
if it fails there, the harness called it wrong and the case is discarded rather
|
|
83
|
+
than counted against the device. Only a call the CPU accepts and the device
|
|
84
|
+
refuses is a finding.
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
obe dump --out card.obl # on each card, once
|
|
88
|
+
obe diff a.obl b.obl # anywhere, with no hardware present
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Measured on an RTX 4090 against a Moore Threads MTT S4000, both on torch 2.2.0.
|
|
92
|
+
Seven hundred callables discovered, 399 valid on the CPU and run on both:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
OPERATOR COVERAGE (399 operators run on both)
|
|
96
|
+
behave the same 272
|
|
97
|
+
numerically different 20
|
|
98
|
+
unavailable on the target 105 (73 distinct operations)
|
|
99
|
+
|
|
100
|
+
WORKS ON THE REFERENCE, NOT ON THE TARGET
|
|
101
|
+
x std median svd qr cholesky det logdet
|
|
102
|
+
x asin asinh acosh atanh sinh cosh log1p
|
|
103
|
+
x lgamma digamma erfc exp2 hypot nansum kthvalue
|
|
104
|
+
and 52 more
|
|
105
|
+
|
|
106
|
+
RUNS ON BOTH, RETURNS A DIFFERENT ANSWER
|
|
107
|
+
! torch.selu MISMATCH 4.83e-02
|
|
108
|
+
! torch.vander MISMATCH 1.00e+00
|
|
109
|
+
! torch.max_pool1d_with_indices SHAPE
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
SELU multiplies positive inputs by 1.0507. On that card it returns them
|
|
113
|
+
unchanged, so every positive activation is five percent low, the negative branch
|
|
114
|
+
is correct, and nothing raises. A model built on it trains, converges, and is
|
|
115
|
+
wrong. That is the case no amount of running the program would have found.
|
|
116
|
+
|
|
117
|
+
Each profile carries a digest over everything it asserts, and a comparison cites
|
|
118
|
+
the digests it was built from, so a report can be checked against the
|
|
119
|
+
measurements it came from and a withdrawn measurement can be named exactly.
|
|
120
|
+
|
|
121
|
+
Profiles accumulate. Ten cards is ten hardware sessions and forty-five pairings,
|
|
122
|
+
and every pairing after the first is a file read.
|
|
123
|
+
|
|
124
|
+
## Making it run, and saying what that cost
|
|
125
|
+
|
|
126
|
+
A missing operator is rarely a missing capability. log1p is log(1+x), asinh is a
|
|
127
|
+
logarithm and a square root, a standard deviation is the square root of a
|
|
128
|
+
variance. The hardware can nearly always do the work; what it lacks is someone to
|
|
129
|
+
write the arrangement down.
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
obe run train.py
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
patches what the device cannot do and runs the program. Three kinds of
|
|
136
|
+
substitution, and every one of them recorded:
|
|
137
|
+
|
|
138
|
+
| | |
|
|
139
|
+
|---|---|
|
|
140
|
+
| rewrite | expressed in operators the device does have, still on the device |
|
|
141
|
+
| fallback | computed on the CPU, which is correct and slow, and the ledger says how slow |
|
|
142
|
+
| correct | the device has the operator, and the operator is wrong |
|
|
143
|
+
|
|
144
|
+
On the MTT S4000 an ordinary piece of scientific PyTorch dies at line 11 with
|
|
145
|
+
`aten::std.correction`. Under `obe run` it completes, having stood in for 66
|
|
146
|
+
operators, and finishes with this:
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
LEDGER: what did not happen the way you wrote it
|
|
150
|
+
|
|
151
|
+
operator how calls seconds
|
|
152
|
+
selu correct 2 0.000
|
|
153
|
+
slogdet fallback 1 0.001
|
|
154
|
+
std rewrite 1 0.000
|
|
155
|
+
median rewrite 1 0.054
|
|
156
|
+
...
|
|
157
|
+
10 call(s) through 9 substituted operator(s), 0.056 s total
|
|
158
|
+
0.001 s of that was spent moving data to the CPU and back
|
|
159
|
+
selu was called 2 time(s); the device kernel is wrong, so the result you
|
|
160
|
+
got is not the result the device would have given
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
The point is not that the program stopped crashing. Anything can stop a program
|
|
164
|
+
crashing. The point is the last line, which a silent compatibility layer would
|
|
165
|
+
never tell you.
|
|
166
|
+
|
|
167
|
+
## When the card and the code are in different places
|
|
168
|
+
|
|
169
|
+
They almost always are. The code is on a laptop or a build server; the card is in
|
|
170
|
+
a machine room with no route out, and nobody has put the code on it yet. That is
|
|
171
|
+
exactly why nobody checks.
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
obe dump --out s4000.obl # on the card, with none of your code present
|
|
175
|
+
scp s4000.obl laptop: # one file, under a megabyte
|
|
176
|
+
obe check ./repo --card s4000.obl # on a box with no accelerator and no torch
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
```
|
|
180
|
+
CODE AGAINST CARD GRADE C
|
|
181
|
+
|
|
182
|
+
code ./normalizing-flows (71 python files)
|
|
183
|
+
card MTT S4000 (musa)
|
|
184
|
+
|
|
185
|
+
7 operations this code calls would fail here, and all of them can be substituted.
|
|
186
|
+
|
|
187
|
+
WILL NOT RUN
|
|
188
|
+
x torch.slogdet MISSING obe can run it on the CPU 4 call sites
|
|
189
|
+
normflows/flows/mixing.py:117
|
|
190
|
+
x torch.std MISSING obe can rewrite it 3 call sites
|
|
191
|
+
normflows/flows/normalization.py:23
|
|
192
|
+
x F.logsigmoid MISSING obe can rewrite it 2 call sites
|
|
193
|
+
x torch.qr MISSING obe can run it on the CPU 2 call sites
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Nobody bought anything, nobody shipped code into a machine room, and nobody
|
|
197
|
+
waited for a slot on a card.
|
|
198
|
+
|
|
199
|
+
## What it checks
|
|
200
|
+
|
|
201
|
+
Fifteen probes, each isolating one assumption that programs make about their
|
|
202
|
+
accelerator and never write down.
|
|
203
|
+
|
|
204
|
+
| group | assumption |
|
|
205
|
+
|---|---|
|
|
206
|
+
| accumulation | summation order, half-precision accumulator width |
|
|
207
|
+
| precision | float32 matmul precision, bfloat16 support |
|
|
208
|
+
| math library | transcendental functions, softmax numerics, normalisation epsilon |
|
|
209
|
+
| parallel ordering | atomic accumulation order, run-to-run stability, reduction over awkward shapes |
|
|
210
|
+
| randomness | the device random stream |
|
|
211
|
+
| selection | tie breaking in sort, topk and argmax |
|
|
212
|
+
| fused kernels | scaled dot-product attention, convolution |
|
|
213
|
+
| memory | layout and striding |
|
|
214
|
+
|
|
215
|
+
Then the same questions are asked of a real transformer: every layer of a forward
|
|
216
|
+
pass, and the loss and parameters across fifty training steps.
|
|
217
|
+
|
|
218
|
+
## How a difference becomes an explanation
|
|
219
|
+
|
|
220
|
+
A layer that disagrees is a symptom. The probe that disagrees in the same way is
|
|
221
|
+
the cause. When a layer diverges, Obelisk names the probes that could explain it
|
|
222
|
+
and that are themselves failing, so the report ends with sentences like *this
|
|
223
|
+
normalisation layer diverges, and the normalisation epsilon probe is also
|
|
224
|
+
diverging* rather than a wall of numbers.
|
|
225
|
+
|
|
226
|
+
## Grades
|
|
227
|
+
|
|
228
|
+
| | |
|
|
229
|
+
|---|---|
|
|
230
|
+
| **A** | bit-identical throughout |
|
|
231
|
+
| **B** | differs, but every difference is inside the tolerance for its dtype |
|
|
232
|
+
| **C** | at least one assumption is broken; usable with care |
|
|
233
|
+
| **D** | a critical assumption is broken, or an operation is missing |
|
|
234
|
+
| **E** | the device does not agree with itself |
|
|
235
|
+
|
|
236
|
+
## Commands
|
|
237
|
+
|
|
238
|
+
| | |
|
|
239
|
+
|---|---|
|
|
240
|
+
| `obe sample` | print a few numbers; run it on two machines and read them side by side |
|
|
241
|
+
| `obe run` | run a program with the gaps filled in, and report every substitution |
|
|
242
|
+
| `obe check` | judge a repository against a card profiled somewhere else |
|
|
243
|
+
| `obe dump` | profile this device completely: operators, dtypes, probes |
|
|
244
|
+
| `obe diff a.obl b.obl` | compare two profiles with no hardware present |
|
|
245
|
+
| `obe record` | measure the reference for one workload |
|
|
246
|
+
| `obe verify --ref` | compare this device against a recorded reference |
|
|
247
|
+
| `obe inspect` | read a repository and predict its risk, before any card exists |
|
|
248
|
+
| `obe migrate` | rewrite for another backend, and report what it could not change |
|
|
249
|
+
| `obe probes` | list the assumptions Obelisk knows how to test |
|
|
250
|
+
| `obe devices` | list the accelerators visible here |
|
|
251
|
+
|
|
252
|
+
A grade is never the whole answer, and the report says what drove it.
|
|
253
|
+
|
|
254
|
+
## Before you have a card
|
|
255
|
+
|
|
256
|
+
```
|
|
257
|
+
obe inspect ./your-repo
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
reads the source and predicts the risk from the assumptions written into it:
|
|
261
|
+
device names, pinned backends, hand-written kernels, precision flags,
|
|
262
|
+
order-dependent accumulations. On nanoGPT it finds 81 of them and returns a
|
|
263
|
+
predicted grade of C. The prediction is written down before the run, so it can
|
|
264
|
+
be held against the result afterwards.
|
|
265
|
+
|
|
266
|
+
```
|
|
267
|
+
obe migrate ./your-repo --target musa --patch port.diff
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
rewrites the mechanical part and, more usefully, reports what it could not safely
|
|
271
|
+
change, including every place the old backend is still named after the rewrite.
|
|
272
|
+
|
|
273
|
+
## Why the blueprint is a text file
|
|
274
|
+
|
|
275
|
+
Reference and target are usually not the same machine, and the interesting one
|
|
276
|
+
usually has no network. A blueprint is about a megabyte of JSON. It travels on a
|
|
277
|
+
USB stick, it can be opened and read by a person, and it carries enough of each
|
|
278
|
+
tensor — a digest, summary statistics, and a deterministic sample of the values —
|
|
279
|
+
that the two sides can be compared elementwise without either tensor ever being
|
|
280
|
+
in the same room as the other.
|
|
281
|
+
|
|
282
|
+
Nothing of your code or your data goes into it. Obelisk measures the device, not
|
|
283
|
+
the workload you are protecting.
|
|
284
|
+
|
|
285
|
+
## Tolerance
|
|
286
|
+
|
|
287
|
+
The headline number for every comparison is the largest absolute disagreement
|
|
288
|
+
divided by the size of the largest value in the reference. Relative error on its
|
|
289
|
+
own is unusable here: two values of 1e-41 that differ by 1e-41 are a hundred
|
|
290
|
+
percent apart and both of them are zero.
|
|
291
|
+
|
|
292
|
+
Quantities that have no excuse to differ at all — a random stream, the order of a
|
|
293
|
+
sort — are held to a stricter standard and must match exactly.
|
|
294
|
+
|
|
295
|
+
## Supported backends
|
|
296
|
+
|
|
297
|
+
CUDA, MUSA (Moore Threads), and CPU. A backend is three small methods; adding one
|
|
298
|
+
does not touch anything else.
|
|
299
|
+
|
|
300
|
+
## License
|
|
301
|
+
|
|
302
|
+
MIT.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Obelisk: environment alignment with a reference baseline.
|
|
2
|
+
|
|
3
|
+
The unit of work is an *assumption*: something a program believes about its
|
|
4
|
+
accelerator that is never written down in the program itself. Obelisk
|
|
5
|
+
enumerates those assumptions, measures each one as a numerical fingerprint on a
|
|
6
|
+
reference device, carries the fingerprints to a target device in a portable
|
|
7
|
+
blueprint, and reports which assumptions survived the move.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
from .fingerprint import fingerprint, compare_fingerprints # noqa: F401
|