counted-float 0.8.4__tar.gz → 0.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {counted_float-0.8.4 → counted_float-0.9.1}/.gitignore +4 -1
- {counted_float-0.8.4 → counted_float-0.9.1}/PKG-INFO +98 -47
- {counted_float-0.8.4 → counted_float-0.9.1}/README.md +97 -46
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/__init__.py +12 -1
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_flops_micro_benchmark.py +1 -1
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_micro_benchmark.py +4 -2
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_time_utils.py +1 -1
- counted_float-0.9.1/counted_float/_core/counting/_builtin_data.py +147 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/config/__init__.py +2 -1
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/config/_config.py +2 -2
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/config/_defaults.py +16 -2
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_base.py +6 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_flop_counts.py +2 -2
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_flop_weights.py +40 -12
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_fpu_specs.py +7 -3
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v7a_cortex_a9.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v8_cortex_a55.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v8_cortex_a76.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v9_cortex_x1.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v9_cortex_x2.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/arm/arm_v9_cortex_x3.json +52 -0
- counted_float-0.9.1/counted_float/_core/data/specs/x86/amd/zen3_r7_5800x.json +54 -0
- counted_float-0.8.4/counted_float/_core/data/specs/amd_zen4_r9_7900x.json → counted_float-0.9.1/counted_float/_core/data/specs/x86/amd/zen4_r9_7900x.json +8 -6
- counted_float-0.9.1/counted_float/_core/data/specs/x86/amd/zen5_r7_9800x3d.json +54 -0
- counted_float-0.9.1/counted_float/_core/data/specs/x86/intel/gen09_coffee_lake.json +57 -0
- counted_float-0.9.1/counted_float/_core/data/specs/x86/intel/gen10_cannon_lake.json +57 -0
- counted_float-0.8.4/counted_float/_core/data/specs/intel_gen11_tiger_lake.json → counted_float-0.9.1/counted_float/_core/data/specs/x86/intel/gen11_tiger_lake.json +5 -0
- counted_float-0.9.1/counted_float/_core/utils/__init__.py +2 -0
- counted_float-0.9.1/counted_float/_core/utils/_geo_mean.py +9 -0
- counted_float-0.9.1/counted_float/_core/utils/_missing_data.py +64 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/config/__init__.py +6 -4
- {counted_float-0.8.4 → counted_float-0.9.1}/pyproject.toml +4 -1
- counted_float-0.8.4/counted_float/_core/counting/_builtin_data.py +0 -25
- counted_float-0.8.4/counted_float/_core/data/__init__.py +0 -0
- counted_float-0.8.4/counted_float/_core/data/benchmarks/__init__.py +0 -0
- counted_float-0.8.4/counted_float/_core/data/specs/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/LICENSE +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_flops_benchmark_suite.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_models.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/compatibility/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/compatibility/_numba.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/compatibility/_strenum.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/_context_managers.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/_counted_float.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/_global_counter.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/__init__.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_flop_type.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_flops_benchmark_result.py +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/counting/models/_fpu_instruction.py +0 -0
- /counted_float-0.8.4/counted_float/_core/data/benchmarks/apple_m3_max.json → /counted_float-0.9.1/counted_float/_core/data/benchmarks/arm/apple/m3_max_macbook_pro_16.json +0 -0
- /counted_float-0.8.4/counted_float/_core/data/benchmarks/intel_i7_1265u.json → /counted_float-0.9.1/counted_float/_core/data/benchmarks/x86/intel/gen12_i7_1265u.json +0 -0
- /counted_float-0.8.4/counted_float/_core/data/benchmarks/intel_i5_7200u.json → /counted_float-0.9.1/counted_float/_core/data/benchmarks/x86/intel/gen7_i5_7200u.json +0 -0
- {counted_float-0.8.4 → counted_float-0.9.1}/counted_float/benchmarking/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: counted-float
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Count floating-point operations in Python code & benchmark relative flop costs.
|
|
5
5
|
Project-URL: Source, https://github.com/bertpl/counted-float
|
|
6
6
|
Project-URL: ChangeLog, https://github.com/bertpl/counted-float/blob/develop/RELEASE_NOTES.md
|
|
@@ -18,9 +18,9 @@ Description-Content-Type: text/markdown
|
|
|
18
18
|
|
|
19
19
|
<!--START_SECTION:images-->
|
|
20
20
|

|
|
21
|
-

|
|
22
|
+

|
|
23
|
+

|
|
24
24
|
<!--END_SECTION:images-->
|
|
25
25
|
|
|
26
26
|
# counted-float
|
|
@@ -167,10 +167,10 @@ and theoretical estimates of the relative cost of different floating point opera
|
|
|
167
167
|
FlopType.SUB [x-y] : 1
|
|
168
168
|
FlopType.MUL [x*y] : 1
|
|
169
169
|
FlopType.DIV [x/y] : 3
|
|
170
|
-
FlopType.SQRT [sqrt(x)] :
|
|
170
|
+
FlopType.SQRT [sqrt(x)] : 3
|
|
171
171
|
FlopType.POW2 [2^x] : 12
|
|
172
|
-
FlopType.LOG2 [log2(x)] :
|
|
173
|
-
FlopType.POW [x^y] :
|
|
172
|
+
FlopType.LOG2 [log2(x)] : 14
|
|
173
|
+
FlopType.POW [x^y] : 33
|
|
174
174
|
}
|
|
175
175
|
```
|
|
176
176
|
These weights will be used by default when extracting total weighted flop costs:
|
|
@@ -189,10 +189,10 @@ with FlopCountingContext() as ctx:
|
|
|
189
189
|
_ = math.log2(cf2)
|
|
190
190
|
|
|
191
191
|
flop_counts = ctx.flop_counts()
|
|
192
|
-
total_cost = flop_counts.total_weighted_cost() # 1 +
|
|
192
|
+
total_cost = flop_counts.total_weighted_cost() # 1 + 33 + 14 = 48
|
|
193
193
|
```
|
|
194
194
|
Note that the `total_weighted_cost` method will use the default flop weights as returned by `get_flop_weights()`. This can be
|
|
195
|
-
overridden by either
|
|
195
|
+
overridden by either configuring different flop weights (see next section) or by setting the `weights` argument of the `total_weighted_cost()` method.
|
|
196
196
|
|
|
197
197
|
|
|
198
198
|
## 2.4. Configuring FLOP weights
|
|
@@ -202,13 +202,15 @@ using the `set_flop_weights()` function, which takes a `FlopWeights` object as a
|
|
|
202
202
|
flop weights that might be obtained using benchmarks run on the target hardware (see later sections).
|
|
203
203
|
|
|
204
204
|
```python
|
|
205
|
-
from counted_float.config import
|
|
205
|
+
from counted_float.config import set_active_flop_weights
|
|
206
206
|
from counted_float import FlopWeights
|
|
207
207
|
|
|
208
|
-
|
|
208
|
+
set_active_flop_weights(weights=FlopWeights(...)) # insert own weights here
|
|
209
209
|
```
|
|
210
210
|
## 2.5. Inspecting built-in data
|
|
211
211
|
|
|
212
|
+
### 2.5.1. Default, pre-aggregated flop weights
|
|
213
|
+
|
|
212
214
|
Built-in empirical, theoretical and consensus built-in flop weights can be inspected using the following functions:
|
|
213
215
|
|
|
214
216
|
```python
|
|
@@ -217,33 +219,64 @@ from counted_float.config import get_default_empirical_flop_weights, get_default
|
|
|
217
219
|
>>> get_default_empirical_flop_weights(rounded=False).show()
|
|
218
220
|
|
|
219
221
|
{
|
|
220
|
-
FlopType.ABS [abs(x)] : 0.
|
|
221
|
-
FlopType.MINUS [-x] : 0.
|
|
222
|
-
FlopType.EQUALS [x==y] : 0.
|
|
223
|
-
FlopType.GTE [x>=y] : 0.
|
|
224
|
-
FlopType.LTE [x<=y] : 0.
|
|
225
|
-
FlopType.CMP_ZERO [x>=0] : 0.
|
|
226
|
-
FlopType.RND [round(x)] :
|
|
227
|
-
FlopType.ADD [x+y] : 0.
|
|
228
|
-
FlopType.SUB [x-y] : 1.
|
|
229
|
-
FlopType.MUL [x*y] : 1.
|
|
230
|
-
FlopType.DIV [x/y] : 3.
|
|
231
|
-
FlopType.SQRT [sqrt(x)] : 2.
|
|
232
|
-
FlopType.POW2 [2^x] : 10.
|
|
233
|
-
FlopType.LOG2 [log2(x)] :
|
|
234
|
-
FlopType.POW [x^y] :
|
|
222
|
+
FlopType.ABS [abs(x)] : 0.90744
|
|
223
|
+
FlopType.MINUS [-x] : 0.80068
|
|
224
|
+
FlopType.EQUALS [x==y] : 0.93532
|
|
225
|
+
FlopType.GTE [x>=y] : 0.94684
|
|
226
|
+
FlopType.LTE [x<=y] : 0.93101
|
|
227
|
+
FlopType.CMP_ZERO [x>=0] : 0.82204
|
|
228
|
+
FlopType.RND [round(x)] : 0.96944
|
|
229
|
+
FlopType.ADD [x+y] : 0.89296
|
|
230
|
+
FlopType.SUB [x-y] : 1.14383
|
|
231
|
+
FlopType.MUL [x*y] : 1.04677
|
|
232
|
+
FlopType.DIV [x/y] : 3.10940
|
|
233
|
+
FlopType.SQRT [sqrt(x)] : 2.56566
|
|
234
|
+
FlopType.POW2 [2^x] : 10.80030
|
|
235
|
+
FlopType.LOG2 [log2(x)] : 16.32770
|
|
236
|
+
FlopType.POW [x^y] : 40.50382
|
|
235
237
|
}
|
|
236
238
|
```
|
|
237
239
|
|
|
238
|
-
These 3 types of built-in weights are defined as follows:
|
|
239
|
-
* `empirical`: geo-mean of the flop weights corresponding to the built-in **benchmarking** results
|
|
240
|
-
* `theoretical`: geo-mean of the flop weights corresponding to the built-in **specification analyses** (FPU instruction latencies)
|
|
241
|
-
* `consensus`: geo-mean of the `empirical` and `theoretical` flop weights
|
|
242
|
-
|
|
243
240
|
The default weights that are configured in the package are the integer-rounded `consensus` weights.
|
|
244
241
|
|
|
242
|
+
### 2.5.2. Custom-aggregated flop weights
|
|
243
|
+
|
|
244
|
+
We can retrieve built-in flop weights in a more fine-grained manner, by custom filtering and the aggregating them with
|
|
245
|
+
the geometric mean.
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
from counted_float.config import get_builtin_flop_weights
|
|
249
|
+
|
|
250
|
+
>>> get_builtin_flop_weights(key_filter="intel").show()
|
|
251
|
+
|
|
252
|
+
{
|
|
253
|
+
FlopType.ABS [abs(x)] : 0.56708
|
|
254
|
+
FlopType.MINUS [-x] : 0.44910
|
|
255
|
+
FlopType.EQUALS [x==y] : 0.89744
|
|
256
|
+
FlopType.GTE [x>=y] : 0.89744
|
|
257
|
+
FlopType.LTE [x<=y] : 0.89744
|
|
258
|
+
FlopType.CMP_ZERO [x>=0] : 0.84762
|
|
259
|
+
FlopType.RND [round(x)] : 2.63592
|
|
260
|
+
FlopType.ADD [x+y] : 0.86616
|
|
261
|
+
FlopType.SUB [x-y] : 1.10411
|
|
262
|
+
FlopType.MUL [x*y] : 1.16515
|
|
263
|
+
FlopType.DIV [x/y] : 4.55230
|
|
264
|
+
FlopType.SQRT [sqrt(x)] : 4.37234
|
|
265
|
+
FlopType.POW2 [2^x] : 14.78792
|
|
266
|
+
FlopType.LOG2 [log2(x)] : 20.51270
|
|
267
|
+
FlopType.POW [x^y] : 40.16390
|
|
268
|
+
}
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
The 3 built-in *default* flop weights are simply presets for the `key_filter` argument:
|
|
272
|
+
* `get_default_empirical_flop_weights()` --> `get_built_in_flop_weights(key_filter="benchmarks")`
|
|
273
|
+
* `get_default_theoretical_flop_weights()` --> `get_built_in_flop_weights(key_filter="specs")`
|
|
274
|
+
* `get_default_consensus_flop_weights()` --> `get_built_in_flop_weights(key_filter="")`
|
|
275
|
+
|
|
245
276
|
# 3. Benchmarking
|
|
246
277
|
|
|
278
|
+
## 3.1. General
|
|
279
|
+
|
|
247
280
|
If the package is installed with the optional `numba` dependency, it provides
|
|
248
281
|
the ability to micro-benchmark floating point operations as follows:
|
|
249
282
|
|
|
@@ -251,22 +284,25 @@ the ability to micro-benchmark floating point operations as follows:
|
|
|
251
284
|
>>> from counted_float.benchmarking import run_flops_benchmark
|
|
252
285
|
>>> results = run_flops_benchmark()
|
|
253
286
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
FlopType.
|
|
258
|
-
FlopType.
|
|
259
|
-
FlopType.
|
|
260
|
-
FlopType.
|
|
261
|
-
FlopType.
|
|
262
|
-
FlopType.
|
|
263
|
-
FlopType.
|
|
264
|
-
FlopType.
|
|
265
|
-
FlopType.
|
|
266
|
-
FlopType.
|
|
267
|
-
FlopType.
|
|
268
|
-
FlopType.
|
|
269
|
-
FlopType.
|
|
287
|
+
Running FLOPS benchmarks using counted-float 0.9.0 ...
|
|
288
|
+
|
|
289
|
+
baseline : wwwwwwwwww.................... 187.97 ns ± 0.52 ns / 1000 flops
|
|
290
|
+
FlopType.ABS [abs(x)] : wwwwwwwwww.................... 307.23 ns ± 8.37 ns / 1000 flops
|
|
291
|
+
FlopType.CMP_ZERO [x>=0] : wwwwwwwwww.................... 301.36 ns ± 7.22 ns / 1000 flops
|
|
292
|
+
FlopType.RND [round(x)] : wwwwwwwwww.................... 302.96 ns ± 8.39 ns / 1000 flops
|
|
293
|
+
FlopType.MINUS [-x] : wwwwwwwwww.................... 304.00 ns ± 7.99 ns / 1000 flops
|
|
294
|
+
FlopType.EQUALS [x==y] : wwwwwwwwww.................... 319.64 ns ± 6.71 ns / 1000 flops
|
|
295
|
+
FlopType.GTE [x>=y] : wwwwwwwwww.................... 325.35 ns ± 9.26 ns / 1000 flops
|
|
296
|
+
FlopType.LTE [x<=y] : wwwwwwwwww.................... 323.17 ns ± 11.45 ns / 1000 flops
|
|
297
|
+
FlopType.ADD [x+y] : wwwwwwwwww.................... 316.96 ns ± 11.11 ns / 1000 flops
|
|
298
|
+
FlopType.SUB [x-y] : wwwwwwwwww.................... 318.59 ns ± 9.36 ns / 1000 flops
|
|
299
|
+
FlopType.MUL [x*y] : wwwwwwwwww.................... 318.11 ns ± 7.16 ns / 1000 flops
|
|
300
|
+
FlopType.SQRT [sqrt(x)] : wwwwwwwwww.................... 449.06 ns ± 2.42 ns / 1000 flops
|
|
301
|
+
FlopType.DIV [x/y] : wwwwwwwwww.................... 483.70 ns ± 2.00 ns / 1000 flops
|
|
302
|
+
FlopType.POW2 [2^x] : wwwwwwwwww.................... 1.77 µs ± 0.00 µs / 1000 flops
|
|
303
|
+
FlopType.LOG2 [log2(x)] : wwwwwwwwww.................... 2.13 µs ± 0.01 µs / 1000 flops
|
|
304
|
+
FlopType.POW [x^y] : wwwwwwwwww.................... 6.53 µs ± 0.00 µs / 1000 flops
|
|
305
|
+
|
|
270
306
|
|
|
271
307
|
>>> results.flop_weights.show()
|
|
272
308
|
|
|
@@ -289,6 +325,21 @@ FlopType.POW [x^y] : wwwwwwwwww.................... 6.55 µ
|
|
|
289
325
|
}
|
|
290
326
|
```
|
|
291
327
|
|
|
328
|
+
## 3.2. Using `uv`
|
|
329
|
+
|
|
330
|
+
There's a lower-threshold way of running benchmarks if you have `uv` installed. Simply install the package including `numba`.
|
|
331
|
+
|
|
332
|
+
```
|
|
333
|
+
uv tool install git+https://github.com/bertpl/counted-float@main[numba] # latest official release
|
|
334
|
+
uv tool install git+https://github.com/bertpl/counted-float@develop[numba] # or latest develop version
|
|
335
|
+
```
|
|
336
|
+
After which you can run the `run_flops_benchmarks` command from the command line:
|
|
337
|
+
```
|
|
338
|
+
run_flops_benchmark
|
|
339
|
+
```
|
|
340
|
+
Final results will be shown as json.
|
|
341
|
+
|
|
342
|
+
|
|
292
343
|
# 4. Known limitations
|
|
293
344
|
|
|
294
345
|
- currently any non-Python-built-in math operations are not counted (e.g. `numpy`)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
<!--START_SECTION:images-->
|
|
2
2
|

|
|
3
|
-

|
|
4
|
+

|
|
5
|
+

|
|
6
6
|
<!--END_SECTION:images-->
|
|
7
7
|
|
|
8
8
|
# counted-float
|
|
@@ -149,10 +149,10 @@ and theoretical estimates of the relative cost of different floating point opera
|
|
|
149
149
|
FlopType.SUB [x-y] : 1
|
|
150
150
|
FlopType.MUL [x*y] : 1
|
|
151
151
|
FlopType.DIV [x/y] : 3
|
|
152
|
-
FlopType.SQRT [sqrt(x)] :
|
|
152
|
+
FlopType.SQRT [sqrt(x)] : 3
|
|
153
153
|
FlopType.POW2 [2^x] : 12
|
|
154
|
-
FlopType.LOG2 [log2(x)] :
|
|
155
|
-
FlopType.POW [x^y] :
|
|
154
|
+
FlopType.LOG2 [log2(x)] : 14
|
|
155
|
+
FlopType.POW [x^y] : 33
|
|
156
156
|
}
|
|
157
157
|
```
|
|
158
158
|
These weights will be used by default when extracting total weighted flop costs:
|
|
@@ -171,10 +171,10 @@ with FlopCountingContext() as ctx:
|
|
|
171
171
|
_ = math.log2(cf2)
|
|
172
172
|
|
|
173
173
|
flop_counts = ctx.flop_counts()
|
|
174
|
-
total_cost = flop_counts.total_weighted_cost() # 1 +
|
|
174
|
+
total_cost = flop_counts.total_weighted_cost() # 1 + 33 + 14 = 48
|
|
175
175
|
```
|
|
176
176
|
Note that the `total_weighted_cost` method will use the default flop weights as returned by `get_flop_weights()`. This can be
|
|
177
|
-
overridden by either
|
|
177
|
+
overridden by either configuring different flop weights (see next section) or by setting the `weights` argument of the `total_weighted_cost()` method.
|
|
178
178
|
|
|
179
179
|
|
|
180
180
|
## 2.4. Configuring FLOP weights
|
|
@@ -184,13 +184,15 @@ using the `set_flop_weights()` function, which takes a `FlopWeights` object as a
|
|
|
184
184
|
flop weights that might be obtained using benchmarks run on the target hardware (see later sections).
|
|
185
185
|
|
|
186
186
|
```python
|
|
187
|
-
from counted_float.config import
|
|
187
|
+
from counted_float.config import set_active_flop_weights
|
|
188
188
|
from counted_float import FlopWeights
|
|
189
189
|
|
|
190
|
-
|
|
190
|
+
set_active_flop_weights(weights=FlopWeights(...)) # insert own weights here
|
|
191
191
|
```
|
|
192
192
|
## 2.5. Inspecting built-in data
|
|
193
193
|
|
|
194
|
+
### 2.5.1. Default, pre-aggregated flop weights
|
|
195
|
+
|
|
194
196
|
Built-in empirical, theoretical and consensus built-in flop weights can be inspected using the following functions:
|
|
195
197
|
|
|
196
198
|
```python
|
|
@@ -199,33 +201,64 @@ from counted_float.config import get_default_empirical_flop_weights, get_default
|
|
|
199
201
|
>>> get_default_empirical_flop_weights(rounded=False).show()
|
|
200
202
|
|
|
201
203
|
{
|
|
202
|
-
FlopType.ABS [abs(x)] : 0.
|
|
203
|
-
FlopType.MINUS [-x] : 0.
|
|
204
|
-
FlopType.EQUALS [x==y] : 0.
|
|
205
|
-
FlopType.GTE [x>=y] : 0.
|
|
206
|
-
FlopType.LTE [x<=y] : 0.
|
|
207
|
-
FlopType.CMP_ZERO [x>=0] : 0.
|
|
208
|
-
FlopType.RND [round(x)] :
|
|
209
|
-
FlopType.ADD [x+y] : 0.
|
|
210
|
-
FlopType.SUB [x-y] : 1.
|
|
211
|
-
FlopType.MUL [x*y] : 1.
|
|
212
|
-
FlopType.DIV [x/y] : 3.
|
|
213
|
-
FlopType.SQRT [sqrt(x)] : 2.
|
|
214
|
-
FlopType.POW2 [2^x] : 10.
|
|
215
|
-
FlopType.LOG2 [log2(x)] :
|
|
216
|
-
FlopType.POW [x^y] :
|
|
204
|
+
FlopType.ABS [abs(x)] : 0.90744
|
|
205
|
+
FlopType.MINUS [-x] : 0.80068
|
|
206
|
+
FlopType.EQUALS [x==y] : 0.93532
|
|
207
|
+
FlopType.GTE [x>=y] : 0.94684
|
|
208
|
+
FlopType.LTE [x<=y] : 0.93101
|
|
209
|
+
FlopType.CMP_ZERO [x>=0] : 0.82204
|
|
210
|
+
FlopType.RND [round(x)] : 0.96944
|
|
211
|
+
FlopType.ADD [x+y] : 0.89296
|
|
212
|
+
FlopType.SUB [x-y] : 1.14383
|
|
213
|
+
FlopType.MUL [x*y] : 1.04677
|
|
214
|
+
FlopType.DIV [x/y] : 3.10940
|
|
215
|
+
FlopType.SQRT [sqrt(x)] : 2.56566
|
|
216
|
+
FlopType.POW2 [2^x] : 10.80030
|
|
217
|
+
FlopType.LOG2 [log2(x)] : 16.32770
|
|
218
|
+
FlopType.POW [x^y] : 40.50382
|
|
217
219
|
}
|
|
218
220
|
```
|
|
219
221
|
|
|
220
|
-
These 3 types of built-in weights are defined as follows:
|
|
221
|
-
* `empirical`: geo-mean of the flop weights corresponding to the built-in **benchmarking** results
|
|
222
|
-
* `theoretical`: geo-mean of the flop weights corresponding to the built-in **specification analyses** (FPU instruction latencies)
|
|
223
|
-
* `consensus`: geo-mean of the `empirical` and `theoretical` flop weights
|
|
224
|
-
|
|
225
222
|
The default weights that are configured in the package are the integer-rounded `consensus` weights.
|
|
226
223
|
|
|
224
|
+
### 2.5.2. Custom-aggregated flop weights
|
|
225
|
+
|
|
226
|
+
We can retrieve built-in flop weights in a more fine-grained manner, by custom filtering and the aggregating them with
|
|
227
|
+
the geometric mean.
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
from counted_float.config import get_builtin_flop_weights
|
|
231
|
+
|
|
232
|
+
>>> get_builtin_flop_weights(key_filter="intel").show()
|
|
233
|
+
|
|
234
|
+
{
|
|
235
|
+
FlopType.ABS [abs(x)] : 0.56708
|
|
236
|
+
FlopType.MINUS [-x] : 0.44910
|
|
237
|
+
FlopType.EQUALS [x==y] : 0.89744
|
|
238
|
+
FlopType.GTE [x>=y] : 0.89744
|
|
239
|
+
FlopType.LTE [x<=y] : 0.89744
|
|
240
|
+
FlopType.CMP_ZERO [x>=0] : 0.84762
|
|
241
|
+
FlopType.RND [round(x)] : 2.63592
|
|
242
|
+
FlopType.ADD [x+y] : 0.86616
|
|
243
|
+
FlopType.SUB [x-y] : 1.10411
|
|
244
|
+
FlopType.MUL [x*y] : 1.16515
|
|
245
|
+
FlopType.DIV [x/y] : 4.55230
|
|
246
|
+
FlopType.SQRT [sqrt(x)] : 4.37234
|
|
247
|
+
FlopType.POW2 [2^x] : 14.78792
|
|
248
|
+
FlopType.LOG2 [log2(x)] : 20.51270
|
|
249
|
+
FlopType.POW [x^y] : 40.16390
|
|
250
|
+
}
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
The 3 built-in *default* flop weights are simply presets for the `key_filter` argument:
|
|
254
|
+
* `get_default_empirical_flop_weights()` --> `get_built_in_flop_weights(key_filter="benchmarks")`
|
|
255
|
+
* `get_default_theoretical_flop_weights()` --> `get_built_in_flop_weights(key_filter="specs")`
|
|
256
|
+
* `get_default_consensus_flop_weights()` --> `get_built_in_flop_weights(key_filter="")`
|
|
257
|
+
|
|
227
258
|
# 3. Benchmarking
|
|
228
259
|
|
|
260
|
+
## 3.1. General
|
|
261
|
+
|
|
229
262
|
If the package is installed with the optional `numba` dependency, it provides
|
|
230
263
|
the ability to micro-benchmark floating point operations as follows:
|
|
231
264
|
|
|
@@ -233,22 +266,25 @@ the ability to micro-benchmark floating point operations as follows:
|
|
|
233
266
|
>>> from counted_float.benchmarking import run_flops_benchmark
|
|
234
267
|
>>> results = run_flops_benchmark()
|
|
235
268
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
FlopType.
|
|
240
|
-
FlopType.
|
|
241
|
-
FlopType.
|
|
242
|
-
FlopType.
|
|
243
|
-
FlopType.
|
|
244
|
-
FlopType.
|
|
245
|
-
FlopType.
|
|
246
|
-
FlopType.
|
|
247
|
-
FlopType.
|
|
248
|
-
FlopType.
|
|
249
|
-
FlopType.
|
|
250
|
-
FlopType.
|
|
251
|
-
FlopType.
|
|
269
|
+
Running FLOPS benchmarks using counted-float 0.9.0 ...
|
|
270
|
+
|
|
271
|
+
baseline : wwwwwwwwww.................... 187.97 ns ± 0.52 ns / 1000 flops
|
|
272
|
+
FlopType.ABS [abs(x)] : wwwwwwwwww.................... 307.23 ns ± 8.37 ns / 1000 flops
|
|
273
|
+
FlopType.CMP_ZERO [x>=0] : wwwwwwwwww.................... 301.36 ns ± 7.22 ns / 1000 flops
|
|
274
|
+
FlopType.RND [round(x)] : wwwwwwwwww.................... 302.96 ns ± 8.39 ns / 1000 flops
|
|
275
|
+
FlopType.MINUS [-x] : wwwwwwwwww.................... 304.00 ns ± 7.99 ns / 1000 flops
|
|
276
|
+
FlopType.EQUALS [x==y] : wwwwwwwwww.................... 319.64 ns ± 6.71 ns / 1000 flops
|
|
277
|
+
FlopType.GTE [x>=y] : wwwwwwwwww.................... 325.35 ns ± 9.26 ns / 1000 flops
|
|
278
|
+
FlopType.LTE [x<=y] : wwwwwwwwww.................... 323.17 ns ± 11.45 ns / 1000 flops
|
|
279
|
+
FlopType.ADD [x+y] : wwwwwwwwww.................... 316.96 ns ± 11.11 ns / 1000 flops
|
|
280
|
+
FlopType.SUB [x-y] : wwwwwwwwww.................... 318.59 ns ± 9.36 ns / 1000 flops
|
|
281
|
+
FlopType.MUL [x*y] : wwwwwwwwww.................... 318.11 ns ± 7.16 ns / 1000 flops
|
|
282
|
+
FlopType.SQRT [sqrt(x)] : wwwwwwwwww.................... 449.06 ns ± 2.42 ns / 1000 flops
|
|
283
|
+
FlopType.DIV [x/y] : wwwwwwwwww.................... 483.70 ns ± 2.00 ns / 1000 flops
|
|
284
|
+
FlopType.POW2 [2^x] : wwwwwwwwww.................... 1.77 µs ± 0.00 µs / 1000 flops
|
|
285
|
+
FlopType.LOG2 [log2(x)] : wwwwwwwwww.................... 2.13 µs ± 0.01 µs / 1000 flops
|
|
286
|
+
FlopType.POW [x^y] : wwwwwwwwww.................... 6.53 µs ± 0.00 µs / 1000 flops
|
|
287
|
+
|
|
252
288
|
|
|
253
289
|
>>> results.flop_weights.show()
|
|
254
290
|
|
|
@@ -271,6 +307,21 @@ FlopType.POW [x^y] : wwwwwwwwww.................... 6.55 µ
|
|
|
271
307
|
}
|
|
272
308
|
```
|
|
273
309
|
|
|
310
|
+
## 3.2. Using `uv`
|
|
311
|
+
|
|
312
|
+
There's a lower-threshold way of running benchmarks if you have `uv` installed. Simply install the package including `numba`.
|
|
313
|
+
|
|
314
|
+
```
|
|
315
|
+
uv tool install git+https://github.com/bertpl/counted-float@main[numba] # latest official release
|
|
316
|
+
uv tool install git+https://github.com/bertpl/counted-float@develop[numba] # or latest develop version
|
|
317
|
+
```
|
|
318
|
+
After which you can run the `run_flops_benchmarks` command from the command line:
|
|
319
|
+
```
|
|
320
|
+
run_flops_benchmark
|
|
321
|
+
```
|
|
322
|
+
Final results will be shown as json.
|
|
323
|
+
|
|
324
|
+
|
|
274
325
|
# 4. Known limitations
|
|
275
326
|
|
|
276
327
|
- currently any non-Python-built-in math operations are not counted (e.g. `numpy`)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from importlib.metadata import version
|
|
2
|
+
|
|
1
3
|
from counted_float._core.counting.models import FlopsBenchmarkResults
|
|
2
4
|
|
|
3
5
|
from ._flops_benchmark_suite import FlopsBenchmarkSuite
|
|
@@ -5,4 +7,13 @@ from ._flops_benchmark_suite import FlopsBenchmarkSuite
|
|
|
5
7
|
|
|
6
8
|
def run_flops_benchmark() -> FlopsBenchmarkResults:
|
|
7
9
|
"""Run the flops benchmark suite with default settings returns a FlopsBenchmarkResults object."""
|
|
8
|
-
|
|
10
|
+
|
|
11
|
+
print()
|
|
12
|
+
print(f"Running FLOPS benchmarks using counted-float {version('counted-float')} ...")
|
|
13
|
+
print()
|
|
14
|
+
|
|
15
|
+
benchmark_results = FlopsBenchmarkSuite().run()
|
|
16
|
+
|
|
17
|
+
print()
|
|
18
|
+
|
|
19
|
+
return benchmark_results
|
|
@@ -33,7 +33,7 @@ class FlopsMicroBenchmark(MicroBenchmark):
|
|
|
33
33
|
"""
|
|
34
34
|
|
|
35
35
|
def __init__(self, name: str, f: Callable, size: int):
|
|
36
|
-
super().__init__(name=name)
|
|
36
|
+
super().__init__(name=name, single_operation=f"{size} flops")
|
|
37
37
|
self.size = size
|
|
38
38
|
self.f = f
|
|
39
39
|
self.n_operations = 0
|
{counted_float-0.8.4 → counted_float-0.9.1}/counted_float/_core/benchmarking/_micro_benchmark.py
RENAMED
|
@@ -23,8 +23,9 @@ class MicroBenchmark(ABC):
|
|
|
23
23
|
|
|
24
24
|
MAX_N_OPERATIONS_FACTOR = 10 # never adjust n_operations by more than this factor (up or down)
|
|
25
25
|
|
|
26
|
-
def __init__(self, name: str):
|
|
26
|
+
def __init__(self, name: str, single_operation: str = "operation"):
|
|
27
27
|
self.name = name
|
|
28
|
+
self.single_operation = single_operation
|
|
28
29
|
|
|
29
30
|
def run_many(
|
|
30
31
|
self, n_runs_total: int = 20, n_runs_warmup: int = 5, n_seconds_per_run_target: float = 0.5
|
|
@@ -74,7 +75,8 @@ class MicroBenchmark(ABC):
|
|
|
74
75
|
benchmark_runs=benchmark_runs,
|
|
75
76
|
)
|
|
76
77
|
stats = benchmark_result.summary_stats()
|
|
77
|
-
|
|
78
|
+
s_time_duration = format_time_durations(nsec_q25=stats.q25, nsec_q50=stats.q50, nsec_q75=stats.q75)
|
|
79
|
+
print(f" {s_time_duration} / {self.single_operation}")
|
|
78
80
|
|
|
79
81
|
# return quantiles
|
|
80
82
|
return benchmark_result
|
|
@@ -14,7 +14,7 @@ def format_time_durations(nsec_q25: float, nsec_q50: float, nsec_q75: float) ->
|
|
|
14
14
|
else:
|
|
15
15
|
formatter = _format_nsec_as_s
|
|
16
16
|
|
|
17
|
-
return f"{formatter(nsec_q50)} ± {formatter((nsec_q75 - nsec_q25) / 2)}
|
|
17
|
+
return f"{formatter(nsec_q50)} ± {formatter((nsec_q75 - nsec_q25) / 2)}"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def _format_nsec_as_ns(nsec: float) -> str:
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from importlib.resources import files
|
|
2
|
+
|
|
3
|
+
from pydantic import ValidationError
|
|
4
|
+
|
|
5
|
+
from counted_float._core.counting.models import FlopsBenchmarkResults, FlopWeights, InstructionLatencies
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# =================================================================================================
|
|
9
|
+
# Main accessor class
|
|
10
|
+
# =================================================================================================
|
|
11
|
+
class BuiltInData:
|
|
12
|
+
"""
|
|
13
|
+
A class that provides access to built-in data for the counted_float package.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
# FlopWeights
|
|
18
|
+
# -------------------------------------------------------------------------
|
|
19
|
+
@classmethod
|
|
20
|
+
def get_flop_weights(cls, key_filter: str = "") -> FlopWeights:
|
|
21
|
+
"""
|
|
22
|
+
Return averaged FlopWeights over all FlopWeights found using get_flop_weights_dict for the provided key_filter.
|
|
23
|
+
|
|
24
|
+
Averaging happens one key-level at a time, which implicitly defines a recursive weighting scheme. At every level
|
|
25
|
+
of aggregation, an attempt is made to impute missing data (if any) to avoid biasing the average towards entries
|
|
26
|
+
with more complete data.
|
|
27
|
+
"""
|
|
28
|
+
flat_flop_weights_dict = cls.get_flop_weights_dict(key_filter)
|
|
29
|
+
if len(flat_flop_weights_dict) == 0:
|
|
30
|
+
raise ValueError(f"No built-in flop weights found for key_filter='{key_filter}'")
|
|
31
|
+
else:
|
|
32
|
+
nested_flop_weights_dict = _flat_to_nested_dict(flat_flop_weights_dict)
|
|
33
|
+
return _computed_nested_average_flop_weights(nested_flop_weights_dict)
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def get_flop_weights_dict(cls, key_filter: str = "") -> dict[str, FlopWeights]:
|
|
37
|
+
"""
|
|
38
|
+
Get the built-in flop weights data as a dict mapping key -> FlopWeights.
|
|
39
|
+
|
|
40
|
+
Keys be .-separated values indicating the path + filename of the source data file, e.g.:
|
|
41
|
+
'benchmarks.arm.apple_m4_pro'
|
|
42
|
+
'specs.x86.intel_core_i9_13900k'
|
|
43
|
+
...
|
|
44
|
+
|
|
45
|
+
:param key_filter: (str, default="") If non-empty, only include entries whose keys contain this substring.
|
|
46
|
+
:return: A dictionary mapping benchmark names to their corresponding FlopsBenchmarkResults.
|
|
47
|
+
"""
|
|
48
|
+
return {
|
|
49
|
+
key: _construct_flop_weights_from_json_str(json_str)
|
|
50
|
+
for key, json_str in _load_json_files_as_dict(files("counted_float._core.data")).items()
|
|
51
|
+
if key_filter in key
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# -------------------------------------------------------------------------
|
|
55
|
+
# Benchmarks
|
|
56
|
+
# -------------------------------------------------------------------------
|
|
57
|
+
@classmethod
|
|
58
|
+
def benchmarks(cls) -> dict[str, FlopsBenchmarkResults]:
|
|
59
|
+
return {
|
|
60
|
+
key: FlopsBenchmarkResults.model_validate_json(json_str)
|
|
61
|
+
for key, json_str in _load_json_files_as_dict(files("counted_float._core.data.benchmarks")).items()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
# -------------------------------------------------------------------------
|
|
65
|
+
# Specs
|
|
66
|
+
# -------------------------------------------------------------------------
|
|
67
|
+
@classmethod
|
|
68
|
+
def specs(cls) -> dict[str, InstructionLatencies]:
|
|
69
|
+
return {
|
|
70
|
+
key: InstructionLatencies.model_validate_json(json_str)
|
|
71
|
+
for key, json_str in _load_json_files_as_dict(files("counted_float._core.data.specs")).items()
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# =================================================================================================
|
|
76
|
+
# Utilities
|
|
77
|
+
# =================================================================================================
|
|
78
|
+
def _computed_nested_average_flop_weights(nested_flop_weights_dict: dict[str, dict | FlopWeights]) -> FlopWeights:
|
|
79
|
+
# make sure all values of the dict are FlopWeights instances
|
|
80
|
+
for key, value in nested_flop_weights_dict.items():
|
|
81
|
+
if isinstance(value, dict):
|
|
82
|
+
nested_flop_weights_dict[key] = _computed_nested_average_flop_weights(value)
|
|
83
|
+
|
|
84
|
+
# now we can average all FlopWeights instances
|
|
85
|
+
return FlopWeights.as_geo_mean(list(nested_flop_weights_dict.values()))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _flat_to_nested_dict(flat_dict: dict) -> dict:
|
|
89
|
+
"""
|
|
90
|
+
Convert a flat dict with .-separated keys to a nested dict.
|
|
91
|
+
E.g. {'a.b.c': 1, 'a.b.d': 2, 'a.e': 3} -> {'a': {'b': {'c': 1, 'd': 2}, 'e': 3}}
|
|
92
|
+
"""
|
|
93
|
+
nested_dict = {}
|
|
94
|
+
for flat_key, value in flat_dict.items():
|
|
95
|
+
keys = flat_key.split(".")
|
|
96
|
+
d = nested_dict
|
|
97
|
+
for key in keys[:-1]:
|
|
98
|
+
d = d.setdefault(key, dict())
|
|
99
|
+
d[keys[-1]] = value
|
|
100
|
+
return nested_dict
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _load_json_files_as_dict(resource_root) -> dict[str, str]:
|
|
104
|
+
"""
|
|
105
|
+
Read all .json files recursively from the given resource root (or the default one) and return
|
|
106
|
+
a dict mapping key -> json_str, where keys are .-separated values indicating the path
|
|
107
|
+
+ filename of the source data file.
|
|
108
|
+
|
|
109
|
+
Example keys: 'benchmarks.arm.apple_m4_pro'
|
|
110
|
+
'specs.x86.intel_core_i9_13900k'
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
# allow both plain & recursive calls
|
|
114
|
+
# if resource_root is None:
|
|
115
|
+
# resource_root = files("counted_float._core.data")
|
|
116
|
+
|
|
117
|
+
# crawl entire folder structure
|
|
118
|
+
result = {}
|
|
119
|
+
for entry in resource_root.iterdir():
|
|
120
|
+
if entry.is_dir():
|
|
121
|
+
sub_dir_json_dict = _load_json_files_as_dict(entry)
|
|
122
|
+
for key, value in sub_dir_json_dict.items():
|
|
123
|
+
result[f"{entry.name}.{key}"] = value
|
|
124
|
+
elif entry.is_file() and entry.name.endswith(".json"):
|
|
125
|
+
result[entry.stem] = entry.read_text(encoding="utf-8")
|
|
126
|
+
return result
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _construct_flop_weights_from_json_str(json_str: str) -> FlopWeights:
|
|
130
|
+
"""
|
|
131
|
+
Construct a FlopWeights instance from a JSON string, where the JSON string can represent either...
|
|
132
|
+
- FlopsBenchmarkResults
|
|
133
|
+
- InstructionLatencies
|
|
134
|
+
:param json_str: (str) JSON string representing either of the aforementioned data structures.
|
|
135
|
+
:return: FlopWeights instance extracted from the input data.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
# try all supported classes, all of which have a .flop_weights property
|
|
139
|
+
for pydantic_cls in [FlopsBenchmarkResults, InstructionLatencies]:
|
|
140
|
+
try:
|
|
141
|
+
obj = pydantic_cls.model_validate_json(json_str)
|
|
142
|
+
return obj.flop_weights
|
|
143
|
+
except ValidationError:
|
|
144
|
+
continue
|
|
145
|
+
|
|
146
|
+
# none of the supported classes worked
|
|
147
|
+
raise ValueError("Input JSON string does not represent a known data structure.")
|