usearch 2.21.4 → 2.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -69
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +8 -8
- package/c/lib.cpp +32 -14
- package/c/test.c +124 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +224 -48
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/README.md +1 -1
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.test.js +117 -94
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -10,52 +10,37 @@
|
|
|
10
10
|
|
|
11
11
|
#include <usearch/index.hpp> // `expected_gt` and macros
|
|
12
12
|
|
|
13
|
-
#if !defined(USEARCH_USE_OPENMP)
|
|
14
|
-
#define USEARCH_USE_OPENMP 0
|
|
15
|
-
#endif
|
|
16
|
-
|
|
17
|
-
#if USEARCH_USE_OPENMP
|
|
18
|
-
#include <omp.h> // `omp_get_num_threads()`
|
|
19
|
-
#endif
|
|
20
|
-
|
|
21
13
|
#if defined(USEARCH_DEFINED_LINUX)
|
|
22
14
|
#include <sys/auxv.h> // `getauxval()`
|
|
23
15
|
#endif
|
|
24
16
|
|
|
25
|
-
#if !defined(
|
|
26
|
-
#
|
|
27
|
-
#define USEARCH_USE_FP16LIB 0
|
|
28
|
-
#elif defined(USEARCH_DEFINED_ARM)
|
|
29
|
-
#include <arm_fp16.h> // `__fp16`
|
|
30
|
-
#define USEARCH_USE_FP16LIB 0
|
|
31
|
-
#else
|
|
32
|
-
#define USEARCH_USE_FP16LIB 1
|
|
33
|
-
#endif
|
|
17
|
+
#if !defined(USEARCH_USE_OPENMP)
|
|
18
|
+
#define USEARCH_USE_OPENMP 0
|
|
34
19
|
#endif
|
|
35
20
|
|
|
36
|
-
#if
|
|
37
|
-
#include <
|
|
21
|
+
#if USEARCH_USE_OPENMP
|
|
22
|
+
#include <omp.h> // `omp_get_num_threads()`
|
|
38
23
|
#endif
|
|
39
24
|
|
|
40
|
-
#if !defined(
|
|
41
|
-
#define
|
|
25
|
+
#if !defined(USEARCH_USE_NUMKONG)
|
|
26
|
+
#define USEARCH_USE_NUMKONG 0
|
|
42
27
|
#endif
|
|
43
28
|
|
|
44
|
-
#if
|
|
29
|
+
#if USEARCH_USE_NUMKONG
|
|
45
30
|
// Propagate the `f16` settings
|
|
46
31
|
#if defined(USEARCH_CAN_COMPILE_FP16) || defined(USEARCH_CAN_COMPILE_FLOAT16)
|
|
47
32
|
#if USEARCH_CAN_COMPILE_FP16 || USEARCH_CAN_COMPILE_FLOAT16
|
|
48
|
-
#define
|
|
33
|
+
#define NK_NATIVE_F16 1
|
|
49
34
|
#else
|
|
50
|
-
#define
|
|
35
|
+
#define NK_NATIVE_F16 0
|
|
51
36
|
#endif
|
|
52
37
|
#endif
|
|
53
38
|
// Propagate the `bf16` settings
|
|
54
39
|
#if defined(USEARCH_CAN_COMPILE_BF16) || defined(USEARCH_CAN_COMPILE_BFLOAT16)
|
|
55
40
|
#if USEARCH_CAN_COMPILE_BF16 || USEARCH_CAN_COMPILE_BFLOAT16
|
|
56
|
-
#define
|
|
41
|
+
#define NK_NATIVE_BF16 1
|
|
57
42
|
#else
|
|
58
|
-
#define
|
|
43
|
+
#define NK_NATIVE_BF16 0
|
|
59
44
|
#endif
|
|
60
45
|
#endif
|
|
61
46
|
// No problem, if some of the functions are unused or undefined
|
|
@@ -71,7 +56,7 @@
|
|
|
71
56
|
#pragma warning(disable : 4101) // "Unused variables"
|
|
72
57
|
#pragma warning(disable : 4068) // "Unknown pragmas", when MSVC tries to read GCC pragmas
|
|
73
58
|
#endif // _MSC_VER
|
|
74
|
-
#include <
|
|
59
|
+
#include <numkong/numkong.h>
|
|
75
60
|
#ifdef _MSC_VER
|
|
76
61
|
#pragma warning(pop)
|
|
77
62
|
#endif // _MSC_VER
|
|
@@ -88,11 +73,19 @@ struct uuid_t {
|
|
|
88
73
|
std::uint8_t octets[16];
|
|
89
74
|
};
|
|
90
75
|
|
|
91
|
-
class f16_bits_t;
|
|
92
76
|
class bf16_bits_t;
|
|
77
|
+
class f16_bits_t;
|
|
78
|
+
class e5m2_bits_t;
|
|
79
|
+
class e4m3_bits_t;
|
|
80
|
+
class e3m2_bits_t;
|
|
81
|
+
class e2m3_bits_t;
|
|
93
82
|
|
|
94
|
-
using f16_t = f16_bits_t;
|
|
95
83
|
using bf16_t = bf16_bits_t;
|
|
84
|
+
using f16_t = f16_bits_t;
|
|
85
|
+
using e5m2_t = e5m2_bits_t;
|
|
86
|
+
using e4m3_t = e4m3_bits_t;
|
|
87
|
+
using e3m2_t = e3m2_bits_t;
|
|
88
|
+
using e2m3_t = e2m3_bits_t;
|
|
96
89
|
|
|
97
90
|
using f64_t = double;
|
|
98
91
|
using f32_t = float;
|
|
@@ -107,6 +100,14 @@ using i32_t = std::int32_t;
|
|
|
107
100
|
using i16_t = std::int16_t;
|
|
108
101
|
using i8_t = std::int8_t;
|
|
109
102
|
|
|
103
|
+
/**
|
|
104
|
+
* @brief Reinterpret-cast between float and uint32 without UB on most compilers.
|
|
105
|
+
*/
|
|
106
|
+
union fu32_t {
|
|
107
|
+
float f;
|
|
108
|
+
std::uint32_t u;
|
|
109
|
+
};
|
|
110
|
+
|
|
110
111
|
/**
|
|
111
112
|
* @brief Enumerates the most commonly used distance metrics, mostly for dense vector representations.
|
|
112
113
|
*/
|
|
@@ -142,11 +143,15 @@ enum class scalar_kind_t : std::uint8_t {
|
|
|
142
143
|
u40_k = 2,
|
|
143
144
|
uuid_k = 3,
|
|
144
145
|
bf16_k = 4,
|
|
146
|
+
// Mini-floats:
|
|
147
|
+
e5m2_k = 5, ///< FP8 IEEE 754: 1 sign + 5 exponent + 2 mantissa, range +/-57344
|
|
148
|
+
e4m3_k = 6, ///< FP8 OCP: 1 sign + 4 exponent + 3 mantissa, range +/-448
|
|
149
|
+
e3m2_k = 8, ///< FP6: 1 sign + 3 exponent + 2 mantissa, range +/-28
|
|
150
|
+
e2m3_k = 7, ///< FP6: 1 sign + 2 exponent + 3 mantissa, range +/-7.5
|
|
145
151
|
// Common:
|
|
146
152
|
f64_k = 10,
|
|
147
153
|
f32_k = 11,
|
|
148
154
|
f16_k = 12,
|
|
149
|
-
f8_k = 13,
|
|
150
155
|
// Common Integral:
|
|
151
156
|
u64_k = 14,
|
|
152
157
|
u32_k = 15,
|
|
@@ -176,6 +181,14 @@ template <typename scalar_at> scalar_kind_t scalar_kind() noexcept {
|
|
|
176
181
|
return scalar_kind_t::f16_k;
|
|
177
182
|
if (std::is_same<scalar_at, bf16_t>())
|
|
178
183
|
return scalar_kind_t::bf16_k;
|
|
184
|
+
if (std::is_same<scalar_at, e5m2_t>())
|
|
185
|
+
return scalar_kind_t::e5m2_k;
|
|
186
|
+
if (std::is_same<scalar_at, e4m3_t>())
|
|
187
|
+
return scalar_kind_t::e4m3_k;
|
|
188
|
+
if (std::is_same<scalar_at, e3m2_t>())
|
|
189
|
+
return scalar_kind_t::e3m2_k;
|
|
190
|
+
if (std::is_same<scalar_at, e2m3_t>())
|
|
191
|
+
return scalar_kind_t::e2m3_k;
|
|
179
192
|
if (std::is_same<scalar_at, i8_t>())
|
|
180
193
|
return scalar_kind_t::i8_k;
|
|
181
194
|
if (std::is_same<scalar_at, u64_t>())
|
|
@@ -251,7 +264,10 @@ inline std::size_t bits_per_scalar(scalar_kind_t scalar_kind) noexcept {
|
|
|
251
264
|
case scalar_kind_t::f16_k: return 16;
|
|
252
265
|
case scalar_kind_t::u8_k: return 8;
|
|
253
266
|
case scalar_kind_t::i8_k: return 8;
|
|
254
|
-
case scalar_kind_t::
|
|
267
|
+
case scalar_kind_t::e5m2_k: return 8;
|
|
268
|
+
case scalar_kind_t::e4m3_k: return 8;
|
|
269
|
+
case scalar_kind_t::e2m3_k: return 8;
|
|
270
|
+
case scalar_kind_t::e3m2_k: return 8;
|
|
255
271
|
default: return 0;
|
|
256
272
|
}
|
|
257
273
|
}
|
|
@@ -277,7 +293,10 @@ inline std::size_t bits_per_scalar_word(scalar_kind_t scalar_kind) noexcept {
|
|
|
277
293
|
case scalar_kind_t::f16_k: return 16;
|
|
278
294
|
case scalar_kind_t::u8_k: return 8;
|
|
279
295
|
case scalar_kind_t::i8_k: return 8;
|
|
280
|
-
case scalar_kind_t::
|
|
296
|
+
case scalar_kind_t::e5m2_k: return 8;
|
|
297
|
+
case scalar_kind_t::e4m3_k: return 8;
|
|
298
|
+
case scalar_kind_t::e2m3_k: return 8;
|
|
299
|
+
case scalar_kind_t::e3m2_k: return 8;
|
|
281
300
|
default: return 0;
|
|
282
301
|
}
|
|
283
302
|
}
|
|
@@ -302,7 +321,10 @@ inline char const* scalar_kind_name(scalar_kind_t scalar_kind) noexcept {
|
|
|
302
321
|
case scalar_kind_t::f16_k: return "f16";
|
|
303
322
|
case scalar_kind_t::u8_k: return "u8";
|
|
304
323
|
case scalar_kind_t::i8_k: return "i8";
|
|
305
|
-
case scalar_kind_t::
|
|
324
|
+
case scalar_kind_t::e5m2_k: return "e5m2";
|
|
325
|
+
case scalar_kind_t::e4m3_k: return "e4m3";
|
|
326
|
+
case scalar_kind_t::e2m3_k: return "e2m3";
|
|
327
|
+
case scalar_kind_t::e3m2_k: return "e3m2";
|
|
306
328
|
default: return "";
|
|
307
329
|
}
|
|
308
330
|
}
|
|
@@ -332,20 +354,30 @@ inline char const* metric_kind_name(metric_kind_t metric) noexcept {
|
|
|
332
354
|
*/
|
|
333
355
|
inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name, std::size_t len) {
|
|
334
356
|
expected_gt<scalar_kind_t> parsed;
|
|
335
|
-
if (str_equals(name, len, "
|
|
336
|
-
parsed.result = scalar_kind_t::f32_k;
|
|
337
|
-
else if (str_equals(name, len, "f64"))
|
|
357
|
+
if (str_equals(name, len, "f64"))
|
|
338
358
|
parsed.result = scalar_kind_t::f64_k;
|
|
339
|
-
else if (str_equals(name, len, "
|
|
340
|
-
parsed.result = scalar_kind_t::
|
|
359
|
+
else if (str_equals(name, len, "f32"))
|
|
360
|
+
parsed.result = scalar_kind_t::f32_k;
|
|
341
361
|
else if (str_equals(name, len, "bf16"))
|
|
342
362
|
parsed.result = scalar_kind_t::bf16_k;
|
|
363
|
+
else if (str_equals(name, len, "f16"))
|
|
364
|
+
parsed.result = scalar_kind_t::f16_k;
|
|
365
|
+
else if (str_equals(name, len, "e5m2"))
|
|
366
|
+
parsed.result = scalar_kind_t::e5m2_k;
|
|
367
|
+
else if (str_equals(name, len, "e4m3"))
|
|
368
|
+
parsed.result = scalar_kind_t::e4m3_k;
|
|
369
|
+
else if (str_equals(name, len, "e3m2"))
|
|
370
|
+
parsed.result = scalar_kind_t::e3m2_k;
|
|
371
|
+
else if (str_equals(name, len, "e2m3"))
|
|
372
|
+
parsed.result = scalar_kind_t::e2m3_k;
|
|
343
373
|
else if (str_equals(name, len, "i8"))
|
|
344
374
|
parsed.result = scalar_kind_t::i8_k;
|
|
375
|
+
else if (str_equals(name, len, "u8"))
|
|
376
|
+
parsed.result = scalar_kind_t::u8_k;
|
|
345
377
|
else if (str_equals(name, len, "b1"))
|
|
346
378
|
parsed.result = scalar_kind_t::b1x8_k;
|
|
347
379
|
else
|
|
348
|
-
parsed.failed("Unknown type, choose: f64, f32, f16,
|
|
380
|
+
parsed.failed("Unknown type, choose: f64, f32, bf16, f16, e5m2, e4m3, e3m2, e2m3, i8, u8, b1");
|
|
349
381
|
return parsed;
|
|
350
382
|
}
|
|
351
383
|
|
|
@@ -396,15 +428,30 @@ inline expected_gt<metric_kind_t> metric_from_name(char const* name) {
|
|
|
396
428
|
* @brief Convenience function to upcast a half-precision floating point number to a single-precision one.
|
|
397
429
|
*/
|
|
398
430
|
inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
399
|
-
#if
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
return
|
|
431
|
+
#if USEARCH_USE_NUMKONG
|
|
432
|
+
nk_f32_t result;
|
|
433
|
+
nk_f16_to_f32_serial((nk_f16_t const*)&u16, &result);
|
|
434
|
+
return result;
|
|
403
435
|
#else
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
std::
|
|
407
|
-
|
|
436
|
+
std::uint32_t sign = (u16 >> 15) & 1;
|
|
437
|
+
std::uint32_t exponent = (u16 >> 10) & 0x1F;
|
|
438
|
+
std::uint32_t mantissa = u16 & 0x03FF;
|
|
439
|
+
fu32_t conv;
|
|
440
|
+
if (exponent == 0) {
|
|
441
|
+
if (mantissa == 0) {
|
|
442
|
+
conv.u = sign << 31;
|
|
443
|
+
} else {
|
|
444
|
+
// Denormal: use FPU normalization trick
|
|
445
|
+
fu32_t temp;
|
|
446
|
+
temp.f = (float)mantissa;
|
|
447
|
+
conv.u = (sign << 31) | (temp.u - 0x0C000000u);
|
|
448
|
+
}
|
|
449
|
+
} else if (exponent == 31) {
|
|
450
|
+
conv.u = (sign << 31) | 0x7F800000u | (mantissa << 13);
|
|
451
|
+
} else {
|
|
452
|
+
conv.u = (sign << 31) | ((exponent + 112u) << 23) | (mantissa << 13);
|
|
453
|
+
}
|
|
454
|
+
return conv.f;
|
|
408
455
|
#endif
|
|
409
456
|
}
|
|
410
457
|
|
|
@@ -412,28 +459,75 @@ inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
|
412
459
|
* @brief Convenience function to downcast a single-precision floating point number to a half-precision one.
|
|
413
460
|
*/
|
|
414
461
|
inline std::uint16_t f32_to_f16(float f32) noexcept {
|
|
415
|
-
#if
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
std::uint16_t result;
|
|
419
|
-
simsimd_f32_to_f16(f32, (simsimd_f16_t*)&result);
|
|
420
|
-
return result;
|
|
421
|
-
#else
|
|
422
|
-
#warning "It's recommended to use SimSIMD and fp16lib for half-precision numerics"
|
|
423
|
-
_Float16 f16 = _Float16(f32);
|
|
462
|
+
#if USEARCH_USE_NUMKONG
|
|
463
|
+
nk_f16_t result;
|
|
464
|
+
nk_f32_to_f16_serial((nk_f32_t const*)&f32, &result);
|
|
424
465
|
std::uint16_t u16;
|
|
425
|
-
std::memcpy(&u16, &
|
|
466
|
+
std::memcpy(&u16, &result, sizeof(u16));
|
|
426
467
|
return u16;
|
|
468
|
+
#else
|
|
469
|
+
fu32_t conv;
|
|
470
|
+
conv.f = f32;
|
|
471
|
+
std::uint32_t sign = (conv.u >> 31) & 1;
|
|
472
|
+
std::uint32_t exponent = (conv.u >> 23) & 0xFF;
|
|
473
|
+
std::uint32_t mantissa = conv.u & 0x007FFFFFu;
|
|
474
|
+
std::uint16_t result;
|
|
475
|
+
if (exponent == 0) {
|
|
476
|
+
result = (std::uint16_t)(sign << 15);
|
|
477
|
+
} else if (exponent == 255) {
|
|
478
|
+
std::uint16_t payload = (std::uint16_t)(mantissa >> 13);
|
|
479
|
+
if (mantissa != 0 && payload == 0)
|
|
480
|
+
payload = 1;
|
|
481
|
+
result = (std::uint16_t)((sign << 15) | 0x7C00 | payload);
|
|
482
|
+
} else if (exponent <= 102) {
|
|
483
|
+
if (exponent == 102 && mantissa > 0)
|
|
484
|
+
result = (std::uint16_t)((sign << 15) | 0x0001);
|
|
485
|
+
else
|
|
486
|
+
result = (std::uint16_t)(sign << 15);
|
|
487
|
+
} else if (exponent < 113) {
|
|
488
|
+
// Denormal range with RNE rounding
|
|
489
|
+
unsigned shift = 113 - exponent;
|
|
490
|
+
unsigned shift_amount = shift + 13;
|
|
491
|
+
std::uint64_t full_mant = 0x00800000ULL | mantissa;
|
|
492
|
+
std::uint32_t mant = (std::uint32_t)(full_mant >> shift_amount);
|
|
493
|
+
std::uint32_t round_bit = (std::uint32_t)((full_mant >> (shift_amount - 1)) & 1);
|
|
494
|
+
std::uint64_t sticky_bits = full_mant & ((1ULL << (shift_amount - 1)) - 1);
|
|
495
|
+
if (round_bit && (sticky_bits || (mant & 1)))
|
|
496
|
+
mant++;
|
|
497
|
+
result = (std::uint16_t)((sign << 15) | mant);
|
|
498
|
+
} else if (exponent < 143) {
|
|
499
|
+
// Normal range with RNE rounding
|
|
500
|
+
std::uint32_t f16_exp = exponent - 112;
|
|
501
|
+
std::uint32_t f16_mant = mantissa >> 13;
|
|
502
|
+
std::uint32_t round_bit = (mantissa >> 12) & 1;
|
|
503
|
+
std::uint32_t sticky_bits = mantissa & 0xFFF;
|
|
504
|
+
if (round_bit && (sticky_bits || (f16_mant & 1))) {
|
|
505
|
+
f16_mant++;
|
|
506
|
+
if (f16_mant > 0x3FF) {
|
|
507
|
+
f16_mant = 0;
|
|
508
|
+
f16_exp++;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
if (f16_exp > 30)
|
|
512
|
+
result = (std::uint16_t)((sign << 15) | 0x7C00);
|
|
513
|
+
else
|
|
514
|
+
result = (std::uint16_t)((sign << 15) | (f16_exp << 10) | f16_mant);
|
|
515
|
+
} else {
|
|
516
|
+
result = (std::uint16_t)((sign << 15) | 0x7C00);
|
|
517
|
+
}
|
|
518
|
+
return result;
|
|
427
519
|
#endif
|
|
428
520
|
}
|
|
429
521
|
|
|
430
522
|
/**
|
|
431
523
|
* @brief Convenience function to upcast a brain-floating point number to a single-precision one.
|
|
432
|
-
* https://github.com/ashvardanian/
|
|
524
|
+
* https://github.com/ashvardanian/NumKong/blob/7e58e9fee9e096238cf29f7c30774fa3dcd0fe85/include/numkong/cast/serial.h#L226-L244
|
|
433
525
|
*/
|
|
434
526
|
inline float bf16_to_f32(std::uint16_t u16) noexcept {
|
|
435
|
-
#if
|
|
436
|
-
|
|
527
|
+
#if USEARCH_USE_NUMKONG
|
|
528
|
+
nk_f32_t result;
|
|
529
|
+
nk_bf16_to_f32_serial((nk_bf16_t const*)&u16, &result);
|
|
530
|
+
return result;
|
|
437
531
|
#else
|
|
438
532
|
union float_or_unsigned_int_t {
|
|
439
533
|
float f;
|
|
@@ -446,13 +540,15 @@ inline float bf16_to_f32(std::uint16_t u16) noexcept {
|
|
|
446
540
|
|
|
447
541
|
/**
|
|
448
542
|
* @brief Convenience function to downcast a single-precision floating point number to a brain-floating point one.
|
|
449
|
-
* https://github.com/ashvardanian/
|
|
543
|
+
* https://github.com/ashvardanian/NumKong/blob/7e58e9fee9e096238cf29f7c30774fa3dcd0fe85/include/numkong/cast/serial.h#L244-L262
|
|
450
544
|
*/
|
|
451
545
|
inline std::uint16_t f32_to_bf16(float f32) noexcept {
|
|
452
|
-
#if
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
546
|
+
#if USEARCH_USE_NUMKONG
|
|
547
|
+
nk_bf16_t result;
|
|
548
|
+
nk_f32_to_bf16_serial((nk_f32_t const*)&f32, &result);
|
|
549
|
+
std::uint16_t u16;
|
|
550
|
+
std::memcpy(&u16, &result, sizeof(u16));
|
|
551
|
+
return u16;
|
|
456
552
|
#else
|
|
457
553
|
union float_or_unsigned_int_t {
|
|
458
554
|
float f;
|
|
@@ -598,6 +694,656 @@ class bf16_bits_t {
|
|
|
598
694
|
initializer(omp_priv = unum::usearch::bf16_bits_t())
|
|
599
695
|
#endif
|
|
600
696
|
|
|
697
|
+
/**
|
|
698
|
+
* @brief Convenience function to upcast an FP8 E5M2 value to single-precision.
|
|
699
|
+
* E5M2: 1 sign + 5 exponent (bias=15) + 2 mantissa, range +/-57344, supports inf/NaN.
|
|
700
|
+
*/
|
|
701
|
+
inline float e5m2_to_f32(std::uint8_t u8) noexcept {
|
|
702
|
+
#if USEARCH_USE_NUMKONG
|
|
703
|
+
nk_f32_t result;
|
|
704
|
+
nk_e5m2_to_f32_serial((nk_e5m2_t const*)&u8, &result);
|
|
705
|
+
return result;
|
|
706
|
+
#else
|
|
707
|
+
// 128-entry LUT for the 7-bit magnitude, sign handled separately.
|
|
708
|
+
static std::uint32_t const lut[128] = {
|
|
709
|
+
0x00000000, 0x37800000, 0x38000000, 0x38400000, // exp=0 sub
|
|
710
|
+
0x38800000, 0x38A00000, 0x38C00000, 0x38E00000, // exp=1
|
|
711
|
+
0x39000000, 0x39200000, 0x39400000, 0x39600000, // exp=2
|
|
712
|
+
0x39800000, 0x39A00000, 0x39C00000, 0x39E00000, // exp=3
|
|
713
|
+
0x3A000000, 0x3A200000, 0x3A400000, 0x3A600000, // exp=4
|
|
714
|
+
0x3A800000, 0x3AA00000, 0x3AC00000, 0x3AE00000, // exp=5
|
|
715
|
+
0x3B000000, 0x3B200000, 0x3B400000, 0x3B600000, // exp=6
|
|
716
|
+
0x3B800000, 0x3BA00000, 0x3BC00000, 0x3BE00000, // exp=7
|
|
717
|
+
0x3C000000, 0x3C200000, 0x3C400000, 0x3C600000, // exp=8
|
|
718
|
+
0x3C800000, 0x3CA00000, 0x3CC00000, 0x3CE00000, // exp=9
|
|
719
|
+
0x3D000000, 0x3D200000, 0x3D400000, 0x3D600000, // exp=10
|
|
720
|
+
0x3D800000, 0x3DA00000, 0x3DC00000, 0x3DE00000, // exp=11
|
|
721
|
+
0x3E000000, 0x3E200000, 0x3E400000, 0x3E600000, // exp=12
|
|
722
|
+
0x3E800000, 0x3EA00000, 0x3EC00000, 0x3EE00000, // exp=13
|
|
723
|
+
0x3F000000, 0x3F200000, 0x3F400000, 0x3F600000, // exp=14
|
|
724
|
+
0x3F800000, 0x3FA00000, 0x3FC00000, 0x3FE00000, // exp=15
|
|
725
|
+
0x40000000, 0x40200000, 0x40400000, 0x40600000, // exp=16
|
|
726
|
+
0x40800000, 0x40A00000, 0x40C00000, 0x40E00000, // exp=17
|
|
727
|
+
0x41000000, 0x41200000, 0x41400000, 0x41600000, // exp=18
|
|
728
|
+
0x41800000, 0x41A00000, 0x41C00000, 0x41E00000, // exp=19
|
|
729
|
+
0x42000000, 0x42200000, 0x42400000, 0x42600000, // exp=20
|
|
730
|
+
0x42800000, 0x42A00000, 0x42C00000, 0x42E00000, // exp=21
|
|
731
|
+
0x43000000, 0x43200000, 0x43400000, 0x43600000, // exp=22
|
|
732
|
+
0x43800000, 0x43A00000, 0x43C00000, 0x43E00000, // exp=23
|
|
733
|
+
0x44000000, 0x44200000, 0x44400000, 0x44600000, // exp=24
|
|
734
|
+
0x44800000, 0x44A00000, 0x44C00000, 0x44E00000, // exp=25
|
|
735
|
+
0x45000000, 0x45200000, 0x45400000, 0x45600000, // exp=26
|
|
736
|
+
0x45800000, 0x45A00000, 0x45C00000, 0x45E00000, // exp=27
|
|
737
|
+
0x46000000, 0x46200000, 0x46400000, 0x46600000, // exp=28
|
|
738
|
+
0x46800000, 0x46A00000, 0x46C00000, 0x46E00000, // exp=29
|
|
739
|
+
0x47000000, 0x47200000, 0x47400000, 0x47600000, // exp=30
|
|
740
|
+
0x7F800000, 0x7FC00000, 0x7FC00000, 0x7FC00000, // inf, nan
|
|
741
|
+
};
|
|
742
|
+
std::uint32_t sign = (std::uint32_t)(u8 & 0x80) << 24;
|
|
743
|
+
fu32_t conv;
|
|
744
|
+
conv.u = sign | lut[u8 & 0x7F];
|
|
745
|
+
return conv.f;
|
|
746
|
+
#endif
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/**
|
|
750
|
+
* @brief Convenience function to downcast a single-precision value to FP8 E5M2.
|
|
751
|
+
* Uses RNE rounding. Overflow → inf, NaN → NaN, subnormals handled.
|
|
752
|
+
*/
|
|
753
|
+
inline std::uint8_t f32_to_e5m2(float f32) noexcept {
|
|
754
|
+
#if USEARCH_USE_NUMKONG
|
|
755
|
+
nk_e5m2_t result;
|
|
756
|
+
nk_f32_to_e5m2_serial((nk_f32_t const*)&f32, &result);
|
|
757
|
+
return result;
|
|
758
|
+
#else
|
|
759
|
+
fu32_t conv;
|
|
760
|
+
conv.f = f32;
|
|
761
|
+
std::uint32_t sign_bit = conv.u >> 31;
|
|
762
|
+
std::uint32_t abs_bits = conv.u & 0x7FFFFFFFu;
|
|
763
|
+
std::uint8_t sign = (std::uint8_t)(sign_bit << 7);
|
|
764
|
+
|
|
765
|
+
// NaN or inf
|
|
766
|
+
if (abs_bits >= 0x7F800000u) {
|
|
767
|
+
std::uint8_t mant = (abs_bits > 0x7F800000u) ? 0x01u : 0x00u;
|
|
768
|
+
return (std::uint8_t)(sign | 0x7Cu | mant);
|
|
769
|
+
}
|
|
770
|
+
if (abs_bits == 0)
|
|
771
|
+
return sign;
|
|
772
|
+
|
|
773
|
+
float abs_x = sign_bit ? -f32 : f32;
|
|
774
|
+
|
|
775
|
+
// Subnormal range: |x| < 2^-14
|
|
776
|
+
if (abs_x < (1.0f / 16384.0f)) {
|
|
777
|
+
float scaled = abs_x * 65536.0f;
|
|
778
|
+
int mant = (int)scaled;
|
|
779
|
+
float frac = scaled - (float)mant;
|
|
780
|
+
if (frac > 0.5f || (frac == 0.5f && (mant & 1)))
|
|
781
|
+
++mant;
|
|
782
|
+
if (mant > 3)
|
|
783
|
+
return (std::uint8_t)(sign | 0x04u);
|
|
784
|
+
return (std::uint8_t)(sign | (std::uint8_t)mant);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
int exp = (int)((abs_bits >> 23) & 0xFFu) - 127;
|
|
788
|
+
std::uint32_t mantissa = abs_bits & 0x7FFFFFu;
|
|
789
|
+
std::uint32_t significand = (1u << 23) | mantissa;
|
|
790
|
+
int shift = 23 - 2;
|
|
791
|
+
std::uint32_t remainder_mask = (1u << shift) - 1;
|
|
792
|
+
std::uint32_t remainder = significand & remainder_mask;
|
|
793
|
+
std::uint32_t halfway = 1u << (shift - 1);
|
|
794
|
+
std::uint32_t significand_rounded = significand >> shift;
|
|
795
|
+
if (remainder > halfway || (remainder == halfway && (significand_rounded & 1)))
|
|
796
|
+
++significand_rounded;
|
|
797
|
+
if (significand_rounded == (1u << 3)) {
|
|
798
|
+
significand_rounded >>= 1;
|
|
799
|
+
++exp;
|
|
800
|
+
}
|
|
801
|
+
if (exp > 15)
|
|
802
|
+
return (std::uint8_t)(sign | 0x7Cu); // overflow → inf
|
|
803
|
+
if (exp < -14) {
|
|
804
|
+
float scaled = abs_x * 65536.0f;
|
|
805
|
+
int mant = (int)scaled;
|
|
806
|
+
float frac = scaled - (float)mant;
|
|
807
|
+
if (frac > 0.5f || (frac == 0.5f && (mant & 1)))
|
|
808
|
+
++mant;
|
|
809
|
+
if (mant > 3)
|
|
810
|
+
return (std::uint8_t)(sign | 0x04u);
|
|
811
|
+
return (std::uint8_t)(sign | (std::uint8_t)mant);
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
std::uint8_t exp_field = (std::uint8_t)(exp + 15);
|
|
815
|
+
std::uint8_t mant_field = (std::uint8_t)(significand_rounded & 0x03u);
|
|
816
|
+
return (std::uint8_t)(sign | (exp_field << 2) | mant_field);
|
|
817
|
+
#endif
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* @brief Convenience function to upcast an FP8 E4M3 value to single-precision.
|
|
822
|
+
* E4M3: 1 sign + 4 exponent (bias=7) + 3 mantissa, range +/-448, no inf.
|
|
823
|
+
*/
|
|
824
|
+
inline float e4m3_to_f32(std::uint8_t u8) noexcept {
|
|
825
|
+
#if USEARCH_USE_NUMKONG
|
|
826
|
+
nk_f32_t result;
|
|
827
|
+
nk_e4m3_to_f32_serial((nk_e4m3_t const*)&u8, &result);
|
|
828
|
+
return result;
|
|
829
|
+
#else
|
|
830
|
+
static std::uint32_t const lut[128] = {
|
|
831
|
+
0x00000000, 0x3B000000, 0x3B800000, 0x3BC00000, 0x3C000000, 0x3C200000, 0x3C400000, 0x3C600000, // exp=0 sub
|
|
832
|
+
0x3C800000, 0x3C900000, 0x3CA00000, 0x3CB00000, 0x3CC00000, 0x3CD00000, 0x3CE00000, 0x3CF00000, // exp=1
|
|
833
|
+
0x3D000000, 0x3D100000, 0x3D200000, 0x3D300000, 0x3D400000, 0x3D500000, 0x3D600000, 0x3D700000, // exp=2
|
|
834
|
+
0x3D800000, 0x3D900000, 0x3DA00000, 0x3DB00000, 0x3DC00000, 0x3DD00000, 0x3DE00000, 0x3DF00000, // exp=3
|
|
835
|
+
0x3E000000, 0x3E100000, 0x3E200000, 0x3E300000, 0x3E400000, 0x3E500000, 0x3E600000, 0x3E700000, // exp=4
|
|
836
|
+
0x3E800000, 0x3E900000, 0x3EA00000, 0x3EB00000, 0x3EC00000, 0x3ED00000, 0x3EE00000, 0x3EF00000, // exp=5
|
|
837
|
+
0x3F000000, 0x3F100000, 0x3F200000, 0x3F300000, 0x3F400000, 0x3F500000, 0x3F600000, 0x3F700000, // exp=6
|
|
838
|
+
0x3F800000, 0x3F900000, 0x3FA00000, 0x3FB00000, 0x3FC00000, 0x3FD00000, 0x3FE00000, 0x3FF00000, // exp=7
|
|
839
|
+
0x40000000, 0x40100000, 0x40200000, 0x40300000, 0x40400000, 0x40500000, 0x40600000, 0x40700000, // exp=8
|
|
840
|
+
0x40800000, 0x40900000, 0x40A00000, 0x40B00000, 0x40C00000, 0x40D00000, 0x40E00000, 0x40F00000, // exp=9
|
|
841
|
+
0x41000000, 0x41100000, 0x41200000, 0x41300000, 0x41400000, 0x41500000, 0x41600000, 0x41700000, // exp=10
|
|
842
|
+
0x41800000, 0x41900000, 0x41A00000, 0x41B00000, 0x41C00000, 0x41D00000, 0x41E00000, 0x41F00000, // exp=11
|
|
843
|
+
0x42000000, 0x42100000, 0x42200000, 0x42300000, 0x42400000, 0x42500000, 0x42600000, 0x42700000, // exp=12
|
|
844
|
+
0x42800000, 0x42900000, 0x42A00000, 0x42B00000, 0x42C00000, 0x42D00000, 0x42E00000, 0x42F00000, // exp=13
|
|
845
|
+
0x43000000, 0x43100000, 0x43200000, 0x43300000, 0x43400000, 0x43500000, 0x43600000, 0x43700000, // exp=14
|
|
846
|
+
0x43800000, 0x43900000, 0x43A00000, 0x43B00000, 0x43C00000, 0x43D00000, 0x43E00000, 0x7FC00000, // exp=15
|
|
847
|
+
};
|
|
848
|
+
std::uint32_t sign = (std::uint32_t)(u8 & 0x80) << 24;
|
|
849
|
+
fu32_t conv;
|
|
850
|
+
conv.u = sign | lut[u8 & 0x7F];
|
|
851
|
+
return conv.f;
|
|
852
|
+
#endif
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* @brief Convenience function to downcast a single-precision value to FP8 E4M3.
|
|
857
|
+
* Uses RNE rounding. Overflow saturates to +/-448 (no inf in E4M3FN).
|
|
858
|
+
*/
|
|
859
|
+
inline std::uint8_t f32_to_e4m3(float f32) noexcept {
|
|
860
|
+
#if USEARCH_USE_NUMKONG
|
|
861
|
+
nk_e4m3_t result;
|
|
862
|
+
nk_f32_to_e4m3_serial((nk_f32_t const*)&f32, &result);
|
|
863
|
+
return result;
|
|
864
|
+
#else
|
|
865
|
+
fu32_t conv;
|
|
866
|
+
conv.f = f32;
|
|
867
|
+
std::uint32_t sign_bit = conv.u >> 31;
|
|
868
|
+
std::uint32_t abs_bits = conv.u & 0x7FFFFFFFu;
|
|
869
|
+
std::uint8_t sign = (std::uint8_t)(sign_bit << 7);
|
|
870
|
+
|
|
871
|
+
// NaN → E4M3FN NaN
|
|
872
|
+
if (abs_bits > 0x7F800000u)
|
|
873
|
+
return (std::uint8_t)(sign | 0x7Fu);
|
|
874
|
+
// Inf → saturate to max (448)
|
|
875
|
+
if (abs_bits == 0x7F800000u)
|
|
876
|
+
return (std::uint8_t)(sign | 0x7Eu);
|
|
877
|
+
if (abs_bits == 0)
|
|
878
|
+
return sign;
|
|
879
|
+
|
|
880
|
+
float abs_x = sign_bit ? -f32 : f32;
|
|
881
|
+
|
|
882
|
+
// Subnormal range: |x| < 2^-6
|
|
883
|
+
if (abs_x < (1.0f / 64.0f)) {
|
|
884
|
+
float scaled = abs_x * 512.0f;
|
|
885
|
+
int mant = (int)scaled;
|
|
886
|
+
float frac = scaled - (float)mant;
|
|
887
|
+
if (frac > 0.5f || (frac == 0.5f && (mant & 1)))
|
|
888
|
+
++mant;
|
|
889
|
+
if (mant > 7)
|
|
890
|
+
return (std::uint8_t)(sign | 0x08u);
|
|
891
|
+
return (std::uint8_t)(sign | (std::uint8_t)mant);
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
int exp = (int)((abs_bits >> 23) & 0xFFu) - 127;
|
|
895
|
+
std::uint32_t mantissa = abs_bits & 0x7FFFFFu;
|
|
896
|
+
std::uint32_t significand = (1u << 23) | mantissa;
|
|
897
|
+
int shift = 23 - 3;
|
|
898
|
+
std::uint32_t remainder_mask = (1u << shift) - 1;
|
|
899
|
+
std::uint32_t remainder = significand & remainder_mask;
|
|
900
|
+
std::uint32_t halfway = 1u << (shift - 1);
|
|
901
|
+
std::uint32_t significand_rounded = significand >> shift;
|
|
902
|
+
if (remainder > halfway || (remainder == halfway && (significand_rounded & 1)))
|
|
903
|
+
++significand_rounded;
|
|
904
|
+
if (significand_rounded == (1u << 4)) {
|
|
905
|
+
significand_rounded >>= 1;
|
|
906
|
+
++exp;
|
|
907
|
+
}
|
|
908
|
+
// Overflow → saturate to max (0x7E = 448)
|
|
909
|
+
if (exp > 8)
|
|
910
|
+
return (std::uint8_t)(sign | 0x7Eu);
|
|
911
|
+
if (exp < -6) {
|
|
912
|
+
float scaled = abs_x * 512.0f;
|
|
913
|
+
int mant = (int)scaled;
|
|
914
|
+
float frac = scaled - (float)mant;
|
|
915
|
+
if (frac > 0.5f || (frac == 0.5f && (mant & 1)))
|
|
916
|
+
++mant;
|
|
917
|
+
if (mant > 7)
|
|
918
|
+
return (std::uint8_t)(sign | 0x08u);
|
|
919
|
+
return (std::uint8_t)(sign | (std::uint8_t)mant);
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
std::uint8_t exp_field = (std::uint8_t)(exp + 7);
|
|
923
|
+
std::uint8_t mant_field = (std::uint8_t)(significand_rounded & 0x07u);
|
|
924
|
+
// Clamp to avoid NaN encoding (0x7F)
|
|
925
|
+
if (exp_field == 15 && mant_field > 6)
|
|
926
|
+
mant_field = 6;
|
|
927
|
+
return (std::uint8_t)(sign | (exp_field << 3) | mant_field);
|
|
928
|
+
#endif
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/**
|
|
932
|
+
* @brief Numeric type for FP8 E5M2 (IEEE 754-like) floating point.
|
|
933
|
+
* 1 sign + 5 exponent + 2 mantissa bits, range +/-57344.
|
|
934
|
+
*/
|
|
935
|
+
class e5m2_bits_t {
|
|
936
|
+
std::uint8_t uint8_{};
|
|
937
|
+
|
|
938
|
+
public:
|
|
939
|
+
inline e5m2_bits_t() noexcept : uint8_(0) {}
|
|
940
|
+
inline e5m2_bits_t(e5m2_bits_t&&) = default;
|
|
941
|
+
inline e5m2_bits_t& operator=(e5m2_bits_t&&) = default;
|
|
942
|
+
inline e5m2_bits_t(e5m2_bits_t const&) = default;
|
|
943
|
+
inline e5m2_bits_t& operator=(e5m2_bits_t const&) = default;
|
|
944
|
+
|
|
945
|
+
inline operator float() const noexcept { return e5m2_to_f32(uint8_); }
|
|
946
|
+
inline explicit operator bool() const noexcept { return e5m2_to_f32(uint8_) > 0.5f; }
|
|
947
|
+
|
|
948
|
+
inline e5m2_bits_t(int v) noexcept : uint8_(f32_to_e5m2(static_cast<float>(v))) {}
|
|
949
|
+
inline e5m2_bits_t(bool v) noexcept : uint8_(f32_to_e5m2(static_cast<float>(v))) {}
|
|
950
|
+
inline e5m2_bits_t(float v) noexcept : uint8_(f32_to_e5m2(v)) {}
|
|
951
|
+
inline e5m2_bits_t(double v) noexcept : uint8_(f32_to_e5m2(static_cast<float>(v))) {}
|
|
952
|
+
|
|
953
|
+
inline bool operator<(e5m2_bits_t const& other) const noexcept { return float(*this) < float(other); }
|
|
954
|
+
|
|
955
|
+
inline e5m2_bits_t operator+(e5m2_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
956
|
+
inline e5m2_bits_t operator-(e5m2_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
957
|
+
inline e5m2_bits_t operator*(e5m2_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
958
|
+
inline e5m2_bits_t operator/(e5m2_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
959
|
+
inline float operator+(float other) const noexcept { return float(*this) + other; }
|
|
960
|
+
inline float operator-(float other) const noexcept { return float(*this) - other; }
|
|
961
|
+
inline float operator*(float other) const noexcept { return float(*this) * other; }
|
|
962
|
+
inline float operator/(float other) const noexcept { return float(*this) / other; }
|
|
963
|
+
inline double operator+(double other) const noexcept { return float(*this) + other; }
|
|
964
|
+
inline double operator-(double other) const noexcept { return float(*this) - other; }
|
|
965
|
+
inline double operator*(double other) const noexcept { return float(*this) * other; }
|
|
966
|
+
inline double operator/(double other) const noexcept { return float(*this) / other; }
|
|
967
|
+
|
|
968
|
+
inline e5m2_bits_t& operator+=(float v) noexcept {
|
|
969
|
+
uint8_ = f32_to_e5m2(v + e5m2_to_f32(uint8_));
|
|
970
|
+
return *this;
|
|
971
|
+
}
|
|
972
|
+
inline e5m2_bits_t& operator-=(float v) noexcept {
|
|
973
|
+
uint8_ = f32_to_e5m2(v - e5m2_to_f32(uint8_));
|
|
974
|
+
return *this;
|
|
975
|
+
}
|
|
976
|
+
inline e5m2_bits_t& operator*=(float v) noexcept {
|
|
977
|
+
uint8_ = f32_to_e5m2(v * e5m2_to_f32(uint8_));
|
|
978
|
+
return *this;
|
|
979
|
+
}
|
|
980
|
+
inline e5m2_bits_t& operator/=(float v) noexcept {
|
|
981
|
+
uint8_ = f32_to_e5m2(v / e5m2_to_f32(uint8_));
|
|
982
|
+
return *this;
|
|
983
|
+
}
|
|
984
|
+
inline e5m2_bits_t& operator=(float v) noexcept {
|
|
985
|
+
uint8_ = f32_to_e5m2(v);
|
|
986
|
+
return *this;
|
|
987
|
+
}
|
|
988
|
+
};
|
|
989
|
+
|
|
990
|
+
/**
|
|
991
|
+
* @brief Numeric type for FP8 E4M3 (OCP) floating point.
|
|
992
|
+
* 1 sign + 4 exponent + 3 mantissa bits, range +/-448.
|
|
993
|
+
*/
|
|
994
|
+
class e4m3_bits_t {
|
|
995
|
+
std::uint8_t uint8_{};
|
|
996
|
+
|
|
997
|
+
public:
|
|
998
|
+
inline e4m3_bits_t() noexcept : uint8_(0) {}
|
|
999
|
+
inline e4m3_bits_t(e4m3_bits_t&&) = default;
|
|
1000
|
+
inline e4m3_bits_t& operator=(e4m3_bits_t&&) = default;
|
|
1001
|
+
inline e4m3_bits_t(e4m3_bits_t const&) = default;
|
|
1002
|
+
inline e4m3_bits_t& operator=(e4m3_bits_t const&) = default;
|
|
1003
|
+
|
|
1004
|
+
inline operator float() const noexcept { return e4m3_to_f32(uint8_); }
|
|
1005
|
+
inline explicit operator bool() const noexcept { return e4m3_to_f32(uint8_) > 0.5f; }
|
|
1006
|
+
|
|
1007
|
+
inline e4m3_bits_t(int v) noexcept : uint8_(f32_to_e4m3(static_cast<float>(v))) {}
|
|
1008
|
+
inline e4m3_bits_t(bool v) noexcept : uint8_(f32_to_e4m3(static_cast<float>(v))) {}
|
|
1009
|
+
inline e4m3_bits_t(float v) noexcept : uint8_(f32_to_e4m3(v)) {}
|
|
1010
|
+
inline e4m3_bits_t(double v) noexcept : uint8_(f32_to_e4m3(static_cast<float>(v))) {}
|
|
1011
|
+
|
|
1012
|
+
inline bool operator<(e4m3_bits_t const& other) const noexcept { return float(*this) < float(other); }
|
|
1013
|
+
|
|
1014
|
+
inline e4m3_bits_t operator+(e4m3_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
1015
|
+
inline e4m3_bits_t operator-(e4m3_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
1016
|
+
inline e4m3_bits_t operator*(e4m3_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
1017
|
+
inline e4m3_bits_t operator/(e4m3_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
1018
|
+
inline float operator+(float other) const noexcept { return float(*this) + other; }
|
|
1019
|
+
inline float operator-(float other) const noexcept { return float(*this) - other; }
|
|
1020
|
+
inline float operator*(float other) const noexcept { return float(*this) * other; }
|
|
1021
|
+
inline float operator/(float other) const noexcept { return float(*this) / other; }
|
|
1022
|
+
inline double operator+(double other) const noexcept { return float(*this) + other; }
|
|
1023
|
+
inline double operator-(double other) const noexcept { return float(*this) - other; }
|
|
1024
|
+
inline double operator*(double other) const noexcept { return float(*this) * other; }
|
|
1025
|
+
inline double operator/(double other) const noexcept { return float(*this) / other; }
|
|
1026
|
+
|
|
1027
|
+
inline e4m3_bits_t& operator+=(float v) noexcept {
|
|
1028
|
+
uint8_ = f32_to_e4m3(v + e4m3_to_f32(uint8_));
|
|
1029
|
+
return *this;
|
|
1030
|
+
}
|
|
1031
|
+
inline e4m3_bits_t& operator-=(float v) noexcept {
|
|
1032
|
+
uint8_ = f32_to_e4m3(v - e4m3_to_f32(uint8_));
|
|
1033
|
+
return *this;
|
|
1034
|
+
}
|
|
1035
|
+
inline e4m3_bits_t& operator*=(float v) noexcept {
|
|
1036
|
+
uint8_ = f32_to_e4m3(v * e4m3_to_f32(uint8_));
|
|
1037
|
+
return *this;
|
|
1038
|
+
}
|
|
1039
|
+
inline e4m3_bits_t& operator/=(float v) noexcept {
|
|
1040
|
+
uint8_ = f32_to_e4m3(v / e4m3_to_f32(uint8_));
|
|
1041
|
+
return *this;
|
|
1042
|
+
}
|
|
1043
|
+
inline e4m3_bits_t& operator=(float v) noexcept {
|
|
1044
|
+
uint8_ = f32_to_e4m3(v);
|
|
1045
|
+
return *this;
|
|
1046
|
+
}
|
|
1047
|
+
};
|
|
1048
|
+
|
|
1049
|
+
/**
|
|
1050
|
+
* @brief Convenience function to upcast an FP6 E2M3 value to single-precision.
|
|
1051
|
+
* E2M3: 1 sign + 2 exponent (bias=1) + 3 mantissa, stored as 0b00SEEMMM, range +/-7.5.
|
|
1052
|
+
*/
|
|
1053
|
+
inline float e2m3_to_f32(std::uint8_t u8) noexcept {
|
|
1054
|
+
#if USEARCH_USE_NUMKONG
|
|
1055
|
+
nk_f32_t result;
|
|
1056
|
+
nk_e2m3_to_f32_serial((nk_e2m3_t const*)&u8, &result);
|
|
1057
|
+
return result;
|
|
1058
|
+
#else
|
|
1059
|
+
static std::uint32_t const lut[64] = {
|
|
1060
|
+
0x00000000, 0x3E000000, 0x3E800000, 0x3EC00000, 0x3F000000, 0x3F200000, 0x3F400000, 0x3F600000, // positive
|
|
1061
|
+
0x3F800000, 0x3F900000, 0x3FA00000, 0x3FB00000, 0x3FC00000, 0x3FD00000, 0x3FE00000, 0x3FF00000, // positive
|
|
1062
|
+
0x40000000, 0x40100000, 0x40200000, 0x40300000, 0x40400000, 0x40500000, 0x40600000, 0x40700000, // positive
|
|
1063
|
+
0x40800000, 0x40900000, 0x40A00000, 0x40B00000, 0x40C00000, 0x40D00000, 0x40E00000, 0x40F00000, // positive
|
|
1064
|
+
0x80000000, 0xBE000000, 0xBE800000, 0xBEC00000, 0xBF000000, 0xBF200000, 0xBF400000, 0xBF600000, // negative
|
|
1065
|
+
0xBF800000, 0xBF900000, 0xBFA00000, 0xBFB00000, 0xBFC00000, 0xBFD00000, 0xBFE00000, 0xBFF00000, // negative
|
|
1066
|
+
0xC0000000, 0xC0100000, 0xC0200000, 0xC0300000, 0xC0400000, 0xC0500000, 0xC0600000, 0xC0700000, // negative
|
|
1067
|
+
0xC0800000, 0xC0900000, 0xC0A00000, 0xC0B00000, 0xC0C00000, 0xC0D00000, 0xC0E00000, 0xC0F00000, // negative
|
|
1068
|
+
};
|
|
1069
|
+
fu32_t conv;
|
|
1070
|
+
conv.u = lut[u8 & 0x3F];
|
|
1071
|
+
return conv.f;
|
|
1072
|
+
#endif
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
/**
|
|
1076
|
+
* @brief Convenience function to downcast a single-precision value to FP6 E2M3.
|
|
1077
|
+
*/
|
|
1078
|
+
inline std::uint8_t f32_to_e2m3(float f32) noexcept {
|
|
1079
|
+
#if USEARCH_USE_NUMKONG
|
|
1080
|
+
nk_e2m3_t result;
|
|
1081
|
+
nk_f32_to_e2m3_serial((nk_f32_t const*)&f32, &result);
|
|
1082
|
+
return result;
|
|
1083
|
+
#else
|
|
1084
|
+
fu32_t conv;
|
|
1085
|
+
conv.f = f32;
|
|
1086
|
+
std::uint32_t sign_bit = conv.u >> 31;
|
|
1087
|
+
std::uint32_t abs_bits = conv.u & 0x7FFFFFFFu;
|
|
1088
|
+
std::uint8_t sign = (std::uint8_t)(sign_bit << 5);
|
|
1089
|
+
if (abs_bits == 0)
|
|
1090
|
+
return sign;
|
|
1091
|
+
float abs_x = sign_bit ? -f32 : f32;
|
|
1092
|
+
// E2M3: bias=1, 2 exp bits, 3 mant bits, max normal = 7.5, min subnormal = 0.125
|
|
1093
|
+
if (abs_x >= 7.5f)
|
|
1094
|
+
return (std::uint8_t)(sign | 0x1Fu); // saturate to max
|
|
1095
|
+
if (abs_x < 0.0625f)
|
|
1096
|
+
return sign; // underflow to zero
|
|
1097
|
+
// Subnormal range: abs_x < 1.0 (min normal = 2^(1-1) = 1.0)
|
|
1098
|
+
if (abs_x < 1.0f) {
|
|
1099
|
+
// Subnormals: value = mant * 2^(-3) = mant/8, so mant = round(abs_x * 8)
|
|
1100
|
+
std::uint32_t mant = (std::uint32_t)(abs_x * 8.0f + 0.5f);
|
|
1101
|
+
if (mant > 7)
|
|
1102
|
+
mant = 7;
|
|
1103
|
+
if (mant == 0)
|
|
1104
|
+
return sign;
|
|
1105
|
+
return (std::uint8_t)(sign | mant);
|
|
1106
|
+
}
|
|
1107
|
+
// Normal range: value = (1 + mant/8) * 2^(exp-1)
|
|
1108
|
+
// Find exponent: exp-1 = floor(log2(abs_x)), so exp = floor(log2(abs_x)) + 1
|
|
1109
|
+
int exp_val;
|
|
1110
|
+
float frac = std::frexp(abs_x, &exp_val); // abs_x = frac * 2^exp_val, frac in [0.5, 1)
|
|
1111
|
+
// frexp returns frac in [0.5, 1), but we want significand in [1, 2)
|
|
1112
|
+
// so significand = frac * 2, and true_exp = exp_val - 1
|
|
1113
|
+
float significand = frac * 2.0f; // [1.0, 2.0)
|
|
1114
|
+
int biased_exp = exp_val; // exp_val - 1 + bias(1) = exp_val
|
|
1115
|
+
if (biased_exp < 1) {
|
|
1116
|
+
// Fell into subnormal, handled above
|
|
1117
|
+
std::uint32_t mant = (std::uint32_t)(abs_x * 8.0f + 0.5f);
|
|
1118
|
+
if (mant > 7)
|
|
1119
|
+
mant = 7;
|
|
1120
|
+
return (std::uint8_t)(sign | mant);
|
|
1121
|
+
}
|
|
1122
|
+
if (biased_exp > 3)
|
|
1123
|
+
biased_exp = 3; // clamp to max exp
|
|
1124
|
+
// Round mantissa: 3 mant bits, significand in [1, 2)
|
|
1125
|
+
std::uint32_t mant = (std::uint32_t)((significand - 1.0f) * 8.0f + 0.5f);
|
|
1126
|
+
if (mant > 7) {
|
|
1127
|
+
mant = 0;
|
|
1128
|
+
biased_exp++;
|
|
1129
|
+
}
|
|
1130
|
+
if (biased_exp > 3)
|
|
1131
|
+
return (std::uint8_t)(sign | 0x1Fu); // overflow
|
|
1132
|
+
return (std::uint8_t)(sign | (biased_exp << 3) | mant);
|
|
1133
|
+
#endif
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/**
|
|
1137
|
+
* @brief Convenience function to upcast an FP6 E3M2 value to single-precision.
|
|
1138
|
+
* E3M2: 1 sign + 3 exponent (bias=3) + 2 mantissa, stored as 0b00SEEEMM, range +/-28.
|
|
1139
|
+
*/
|
|
1140
|
+
inline float e3m2_to_f32(std::uint8_t u8) noexcept {
|
|
1141
|
+
#if USEARCH_USE_NUMKONG
|
|
1142
|
+
nk_f32_t result;
|
|
1143
|
+
nk_e3m2_to_f32_serial((nk_e3m2_t const*)&u8, &result);
|
|
1144
|
+
return result;
|
|
1145
|
+
#else
|
|
1146
|
+
static std::uint32_t const lut[64] = {
|
|
1147
|
+
0x00000000, 0x3D800000, 0x3E000000, 0x3E400000, 0x3E800000, 0x3EA00000, 0x3EC00000, 0x3EE00000, // positive
|
|
1148
|
+
0x3F000000, 0x3F200000, 0x3F400000, 0x3F600000, 0x3F800000, 0x3FA00000, 0x3FC00000, 0x3FE00000, // positive
|
|
1149
|
+
0x40000000, 0x40200000, 0x40400000, 0x40600000, 0x40800000, 0x40A00000, 0x40C00000, 0x40E00000, // positive
|
|
1150
|
+
0x41000000, 0x41200000, 0x41400000, 0x41600000, 0x41800000, 0x41A00000, 0x41C00000, 0x41E00000, // positive
|
|
1151
|
+
0x80000000, 0xBD800000, 0xBE000000, 0xBE400000, 0xBE800000, 0xBEA00000, 0xBEC00000, 0xBEE00000, // negative
|
|
1152
|
+
0xBF000000, 0xBF200000, 0xBF400000, 0xBF600000, 0xBF800000, 0xBFA00000, 0xBFC00000, 0xBFE00000, // negative
|
|
1153
|
+
0xC0000000, 0xC0200000, 0xC0400000, 0xC0600000, 0xC0800000, 0xC0A00000, 0xC0C00000, 0xC0E00000, // negative
|
|
1154
|
+
0xC1000000, 0xC1200000, 0xC1400000, 0xC1600000, 0xC1800000, 0xC1A00000, 0xC1C00000, 0xC1E00000, // negative
|
|
1155
|
+
};
|
|
1156
|
+
fu32_t conv;
|
|
1157
|
+
conv.u = lut[u8 & 0x3F];
|
|
1158
|
+
return conv.f;
|
|
1159
|
+
#endif
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
/**
|
|
1163
|
+
* @brief Convenience function to downcast a single-precision value to FP6 E3M2.
|
|
1164
|
+
*/
|
|
1165
|
+
inline std::uint8_t f32_to_e3m2(float f32) noexcept {
|
|
1166
|
+
#if USEARCH_USE_NUMKONG
|
|
1167
|
+
nk_e3m2_t result;
|
|
1168
|
+
nk_f32_to_e3m2_serial((nk_f32_t const*)&f32, &result);
|
|
1169
|
+
return result;
|
|
1170
|
+
#else
|
|
1171
|
+
fu32_t conv;
|
|
1172
|
+
conv.f = f32;
|
|
1173
|
+
std::uint32_t sign_bit = conv.u >> 31;
|
|
1174
|
+
std::uint32_t abs_bits = conv.u & 0x7FFFFFFFu;
|
|
1175
|
+
std::uint8_t sign = (std::uint8_t)(sign_bit << 5);
|
|
1176
|
+
if (abs_bits == 0)
|
|
1177
|
+
return sign;
|
|
1178
|
+
float abs_x = sign_bit ? -f32 : f32;
|
|
1179
|
+
// E3M2: bias=3, 3 exp bits, 2 mant bits, max normal = 28.0, min subnormal = 0.0625
|
|
1180
|
+
if (abs_x >= 28.0f)
|
|
1181
|
+
return (std::uint8_t)(sign | 0x1Fu); // saturate to max
|
|
1182
|
+
if (abs_x < 0.03125f)
|
|
1183
|
+
return sign; // underflow to zero
|
|
1184
|
+
// Subnormal range: abs_x < 0.25 (min normal = 2^(1-3) = 0.25)
|
|
1185
|
+
if (abs_x < 0.25f) {
|
|
1186
|
+
// Subnormals: value = mant * 2^(-4) = mant/16, so mant = round(abs_x * 16)
|
|
1187
|
+
std::uint32_t mant = (std::uint32_t)(abs_x * 16.0f + 0.5f);
|
|
1188
|
+
if (mant > 3)
|
|
1189
|
+
mant = 3;
|
|
1190
|
+
if (mant == 0)
|
|
1191
|
+
return sign;
|
|
1192
|
+
return (std::uint8_t)(sign | mant);
|
|
1193
|
+
}
|
|
1194
|
+
// Normal range: value = (1 + mant/4) * 2^(exp-3)
|
|
1195
|
+
int exp_val;
|
|
1196
|
+
float frac = std::frexp(abs_x, &exp_val);
|
|
1197
|
+
float significand = frac * 2.0f;
|
|
1198
|
+
int biased_exp = exp_val - 1 + 3; // true_exp = exp_val - 1, biased = true_exp + 3
|
|
1199
|
+
if (biased_exp < 1) {
|
|
1200
|
+
std::uint32_t mant = (std::uint32_t)(abs_x * 16.0f + 0.5f);
|
|
1201
|
+
if (mant > 3)
|
|
1202
|
+
mant = 3;
|
|
1203
|
+
return (std::uint8_t)(sign | mant);
|
|
1204
|
+
}
|
|
1205
|
+
if (biased_exp > 7)
|
|
1206
|
+
biased_exp = 7;
|
|
1207
|
+
std::uint32_t mant = (std::uint32_t)((significand - 1.0f) * 4.0f + 0.5f);
|
|
1208
|
+
if (mant > 3) {
|
|
1209
|
+
mant = 0;
|
|
1210
|
+
biased_exp++;
|
|
1211
|
+
}
|
|
1212
|
+
if (biased_exp > 7)
|
|
1213
|
+
return (std::uint8_t)(sign | 0x1Fu);
|
|
1214
|
+
return (std::uint8_t)(sign | (biased_exp << 2) | mant);
|
|
1215
|
+
#endif
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
/**
|
|
1219
|
+
* @brief Numeric type for FP6 E2M3 floating point.
|
|
1220
|
+
* 1 sign + 2 exponent + 3 mantissa bits, stored as 0b00SEEMMM, range +/-7.5.
|
|
1221
|
+
*/
|
|
1222
|
+
class e2m3_bits_t {
|
|
1223
|
+
std::uint8_t uint8_{};
|
|
1224
|
+
|
|
1225
|
+
public:
|
|
1226
|
+
inline e2m3_bits_t() noexcept : uint8_(0) {}
|
|
1227
|
+
inline e2m3_bits_t(e2m3_bits_t&&) = default;
|
|
1228
|
+
inline e2m3_bits_t& operator=(e2m3_bits_t&&) = default;
|
|
1229
|
+
inline e2m3_bits_t(e2m3_bits_t const&) = default;
|
|
1230
|
+
inline e2m3_bits_t& operator=(e2m3_bits_t const&) = default;
|
|
1231
|
+
|
|
1232
|
+
inline operator float() const noexcept { return e2m3_to_f32(uint8_); }
|
|
1233
|
+
inline explicit operator bool() const noexcept { return e2m3_to_f32(uint8_) > 0.5f; }
|
|
1234
|
+
|
|
1235
|
+
inline e2m3_bits_t(int v) noexcept : uint8_(f32_to_e2m3(static_cast<float>(v))) {}
|
|
1236
|
+
inline e2m3_bits_t(bool v) noexcept : uint8_(f32_to_e2m3(static_cast<float>(v))) {}
|
|
1237
|
+
inline e2m3_bits_t(float v) noexcept : uint8_(f32_to_e2m3(v)) {}
|
|
1238
|
+
inline e2m3_bits_t(double v) noexcept : uint8_(f32_to_e2m3(static_cast<float>(v))) {}
|
|
1239
|
+
|
|
1240
|
+
inline bool operator<(e2m3_bits_t const& other) const noexcept { return float(*this) < float(other); }
|
|
1241
|
+
|
|
1242
|
+
inline e2m3_bits_t operator+(e2m3_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
1243
|
+
inline e2m3_bits_t operator-(e2m3_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
1244
|
+
inline e2m3_bits_t operator*(e2m3_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
1245
|
+
inline e2m3_bits_t operator/(e2m3_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
1246
|
+
inline float operator+(float other) const noexcept { return float(*this) + other; }
|
|
1247
|
+
inline float operator-(float other) const noexcept { return float(*this) - other; }
|
|
1248
|
+
inline float operator*(float other) const noexcept { return float(*this) * other; }
|
|
1249
|
+
inline float operator/(float other) const noexcept { return float(*this) / other; }
|
|
1250
|
+
inline double operator+(double other) const noexcept { return float(*this) + other; }
|
|
1251
|
+
inline double operator-(double other) const noexcept { return float(*this) - other; }
|
|
1252
|
+
inline double operator*(double other) const noexcept { return float(*this) * other; }
|
|
1253
|
+
inline double operator/(double other) const noexcept { return float(*this) / other; }
|
|
1254
|
+
|
|
1255
|
+
inline e2m3_bits_t& operator+=(float v) noexcept {
|
|
1256
|
+
uint8_ = f32_to_e2m3(v + e2m3_to_f32(uint8_));
|
|
1257
|
+
return *this;
|
|
1258
|
+
}
|
|
1259
|
+
inline e2m3_bits_t& operator-=(float v) noexcept {
|
|
1260
|
+
uint8_ = f32_to_e2m3(v - e2m3_to_f32(uint8_));
|
|
1261
|
+
return *this;
|
|
1262
|
+
}
|
|
1263
|
+
inline e2m3_bits_t& operator*=(float v) noexcept {
|
|
1264
|
+
uint8_ = f32_to_e2m3(v * e2m3_to_f32(uint8_));
|
|
1265
|
+
return *this;
|
|
1266
|
+
}
|
|
1267
|
+
inline e2m3_bits_t& operator/=(float v) noexcept {
|
|
1268
|
+
uint8_ = f32_to_e2m3(v / e2m3_to_f32(uint8_));
|
|
1269
|
+
return *this;
|
|
1270
|
+
}
|
|
1271
|
+
inline e2m3_bits_t& operator=(float v) noexcept {
|
|
1272
|
+
uint8_ = f32_to_e2m3(v);
|
|
1273
|
+
return *this;
|
|
1274
|
+
}
|
|
1275
|
+
};
|
|
1276
|
+
|
|
1277
|
+
/**
|
|
1278
|
+
* @brief Numeric type for FP6 E3M2 floating point.
|
|
1279
|
+
* 1 sign + 3 exponent + 2 mantissa bits, stored as 0b00SEEEMM, range +/-28.
|
|
1280
|
+
*/
|
|
1281
|
+
class e3m2_bits_t {
|
|
1282
|
+
std::uint8_t uint8_{};
|
|
1283
|
+
|
|
1284
|
+
public:
|
|
1285
|
+
inline e3m2_bits_t() noexcept : uint8_(0) {}
|
|
1286
|
+
inline e3m2_bits_t(e3m2_bits_t&&) = default;
|
|
1287
|
+
inline e3m2_bits_t& operator=(e3m2_bits_t&&) = default;
|
|
1288
|
+
inline e3m2_bits_t(e3m2_bits_t const&) = default;
|
|
1289
|
+
inline e3m2_bits_t& operator=(e3m2_bits_t const&) = default;
|
|
1290
|
+
|
|
1291
|
+
inline operator float() const noexcept { return e3m2_to_f32(uint8_); }
|
|
1292
|
+
inline explicit operator bool() const noexcept { return e3m2_to_f32(uint8_) > 0.5f; }
|
|
1293
|
+
|
|
1294
|
+
inline e3m2_bits_t(int v) noexcept : uint8_(f32_to_e3m2(static_cast<float>(v))) {}
|
|
1295
|
+
inline e3m2_bits_t(bool v) noexcept : uint8_(f32_to_e3m2(static_cast<float>(v))) {}
|
|
1296
|
+
inline e3m2_bits_t(float v) noexcept : uint8_(f32_to_e3m2(v)) {}
|
|
1297
|
+
inline e3m2_bits_t(double v) noexcept : uint8_(f32_to_e3m2(static_cast<float>(v))) {}
|
|
1298
|
+
|
|
1299
|
+
inline bool operator<(e3m2_bits_t const& other) const noexcept { return float(*this) < float(other); }
|
|
1300
|
+
|
|
1301
|
+
inline e3m2_bits_t operator+(e3m2_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
1302
|
+
inline e3m2_bits_t operator-(e3m2_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
1303
|
+
inline e3m2_bits_t operator*(e3m2_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
1304
|
+
inline e3m2_bits_t operator/(e3m2_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
1305
|
+
inline float operator+(float other) const noexcept { return float(*this) + other; }
|
|
1306
|
+
inline float operator-(float other) const noexcept { return float(*this) - other; }
|
|
1307
|
+
inline float operator*(float other) const noexcept { return float(*this) * other; }
|
|
1308
|
+
inline float operator/(float other) const noexcept { return float(*this) / other; }
|
|
1309
|
+
inline double operator+(double other) const noexcept { return float(*this) + other; }
|
|
1310
|
+
inline double operator-(double other) const noexcept { return float(*this) - other; }
|
|
1311
|
+
inline double operator*(double other) const noexcept { return float(*this) * other; }
|
|
1312
|
+
inline double operator/(double other) const noexcept { return float(*this) / other; }
|
|
1313
|
+
|
|
1314
|
+
inline e3m2_bits_t& operator+=(float v) noexcept {
|
|
1315
|
+
uint8_ = f32_to_e3m2(v + e3m2_to_f32(uint8_));
|
|
1316
|
+
return *this;
|
|
1317
|
+
}
|
|
1318
|
+
inline e3m2_bits_t& operator-=(float v) noexcept {
|
|
1319
|
+
uint8_ = f32_to_e3m2(v - e3m2_to_f32(uint8_));
|
|
1320
|
+
return *this;
|
|
1321
|
+
}
|
|
1322
|
+
inline e3m2_bits_t& operator*=(float v) noexcept {
|
|
1323
|
+
uint8_ = f32_to_e3m2(v * e3m2_to_f32(uint8_));
|
|
1324
|
+
return *this;
|
|
1325
|
+
}
|
|
1326
|
+
inline e3m2_bits_t& operator/=(float v) noexcept {
|
|
1327
|
+
uint8_ = f32_to_e3m2(v / e3m2_to_f32(uint8_));
|
|
1328
|
+
return *this;
|
|
1329
|
+
}
|
|
1330
|
+
inline e3m2_bits_t& operator=(float v) noexcept {
|
|
1331
|
+
uint8_ = f32_to_e3m2(v);
|
|
1332
|
+
return *this;
|
|
1333
|
+
}
|
|
1334
|
+
};
|
|
1335
|
+
|
|
1336
|
+
#if USEARCH_USE_OPENMP
|
|
1337
|
+
#pragma omp declare reduction(+ : unum::usearch::e5m2_bits_t : omp_out = omp_out + omp_in) \
|
|
1338
|
+
initializer(omp_priv = unum::usearch::e5m2_bits_t())
|
|
1339
|
+
#pragma omp declare reduction(+ : unum::usearch::e4m3_bits_t : omp_out = omp_out + omp_in) \
|
|
1340
|
+
initializer(omp_priv = unum::usearch::e4m3_bits_t())
|
|
1341
|
+
#pragma omp declare reduction(+ : unum::usearch::e2m3_bits_t : omp_out = omp_out + omp_in) \
|
|
1342
|
+
initializer(omp_priv = unum::usearch::e2m3_bits_t())
|
|
1343
|
+
#pragma omp declare reduction(+ : unum::usearch::e3m2_bits_t : omp_out = omp_out + omp_in) \
|
|
1344
|
+
initializer(omp_priv = unum::usearch::e3m2_bits_t())
|
|
1345
|
+
#endif
|
|
1346
|
+
|
|
601
1347
|
/**
|
|
602
1348
|
* @brief An STL-based executor or a "thread-pool" for parallel execution.
|
|
603
1349
|
* Isn't efficient for small batches, as it recreates the threads on every call.
|
|
@@ -648,6 +1394,9 @@ class executor_stl_t {
|
|
|
648
1394
|
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
649
1395
|
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
650
1396
|
new (&threads_pool[thread_idx - 1]) jthread_t([=]() {
|
|
1397
|
+
#if USEARCH_USE_NUMKONG
|
|
1398
|
+
nk_configure_thread_(nk_capabilities());
|
|
1399
|
+
#endif
|
|
651
1400
|
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
652
1401
|
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread); ++task_idx)
|
|
653
1402
|
thread_aware_function(thread_idx, task_idx);
|
|
@@ -674,6 +1423,9 @@ class executor_stl_t {
|
|
|
674
1423
|
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
675
1424
|
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
676
1425
|
new (&threads_pool[thread_idx - 1]) jthread_t([=, &stop]() {
|
|
1426
|
+
#if USEARCH_USE_NUMKONG
|
|
1427
|
+
nk_configure_thread_(nk_capabilities());
|
|
1428
|
+
#endif
|
|
677
1429
|
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
678
1430
|
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread) &&
|
|
679
1431
|
!stop.load(std::memory_order_relaxed);
|
|
@@ -700,7 +1452,12 @@ class executor_stl_t {
|
|
|
700
1452
|
return thread_aware_function(0);
|
|
701
1453
|
buffer_gt<jthread_t> threads_pool(threads_count_ - 1);
|
|
702
1454
|
for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx)
|
|
703
|
-
new (&threads_pool[thread_idx - 1]) jthread_t([=]() {
|
|
1455
|
+
new (&threads_pool[thread_idx - 1]) jthread_t([=]() {
|
|
1456
|
+
#if USEARCH_USE_NUMKONG
|
|
1457
|
+
nk_configure_thread_(nk_capabilities());
|
|
1458
|
+
#endif
|
|
1459
|
+
thread_aware_function(thread_idx);
|
|
1460
|
+
});
|
|
704
1461
|
thread_aware_function(0);
|
|
705
1462
|
}
|
|
706
1463
|
};
|
|
@@ -709,7 +1466,7 @@ class executor_stl_t {
|
|
|
709
1466
|
|
|
710
1467
|
/**
|
|
711
1468
|
* @brief An OpenMP-based executor or a "thread-pool" for parallel execution.
|
|
712
|
-
* Is the preferred implementation, when available, and
|
|
1469
|
+
* Is the preferred implementation, when available, and target environment has OpenMP.
|
|
713
1470
|
*/
|
|
714
1471
|
class executor_openmp_t {
|
|
715
1472
|
public:
|
|
@@ -718,6 +1475,14 @@ class executor_openmp_t {
|
|
|
718
1475
|
*/
|
|
719
1476
|
executor_openmp_t(std::size_t threads_count = 0) noexcept {
|
|
720
1477
|
omp_set_num_threads(static_cast<int>(threads_count ? threads_count : std::thread::hardware_concurrency()));
|
|
1478
|
+
#if USEARCH_USE_NUMKONG
|
|
1479
|
+
nk_capability_t caps = nk_capabilities();
|
|
1480
|
+
nk_configure_thread(caps);
|
|
1481
|
+
#pragma omp parallel
|
|
1482
|
+
{
|
|
1483
|
+
nk_configure_thread(caps);
|
|
1484
|
+
}
|
|
1485
|
+
#endif
|
|
721
1486
|
}
|
|
722
1487
|
|
|
723
1488
|
/**
|
|
@@ -854,7 +1619,8 @@ class page_allocator_t {
|
|
|
854
1619
|
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
855
1620
|
return (byte_t*)(::VirtualAlloc(NULL, count_bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
|
|
856
1621
|
#else
|
|
857
|
-
|
|
1622
|
+
auto* result = mmap(NULL, count_bytes, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
1623
|
+
return (result == MAP_FAILED) ? nullptr : (byte_t*)result;
|
|
858
1624
|
#endif
|
|
859
1625
|
}
|
|
860
1626
|
|
|
@@ -889,6 +1655,7 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
889
1655
|
std::size_t last_usage_ = head_size();
|
|
890
1656
|
std::size_t last_capacity_ = min_capacity();
|
|
891
1657
|
std::size_t wasted_space_ = 0;
|
|
1658
|
+
std::size_t total_allocated_ = 0;
|
|
892
1659
|
|
|
893
1660
|
public:
|
|
894
1661
|
using value_type = byte_t;
|
|
@@ -899,13 +1666,15 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
899
1666
|
memory_mapping_allocator_gt() = default;
|
|
900
1667
|
memory_mapping_allocator_gt(memory_mapping_allocator_gt&& other) noexcept
|
|
901
1668
|
: last_arena_(exchange(other.last_arena_, nullptr)), last_usage_(exchange(other.last_usage_, 0)),
|
|
902
|
-
last_capacity_(exchange(other.last_capacity_, 0)), wasted_space_(exchange(other.wasted_space_, 0))
|
|
1669
|
+
last_capacity_(exchange(other.last_capacity_, 0)), wasted_space_(exchange(other.wasted_space_, 0)),
|
|
1670
|
+
total_allocated_(exchange(other.total_allocated_, 0)) {}
|
|
903
1671
|
|
|
904
1672
|
memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt&& other) noexcept {
|
|
905
1673
|
std::swap(last_arena_, other.last_arena_);
|
|
906
1674
|
std::swap(last_usage_, other.last_usage_);
|
|
907
1675
|
std::swap(last_capacity_, other.last_capacity_);
|
|
908
1676
|
std::swap(wasted_space_, other.wasted_space_);
|
|
1677
|
+
std::swap(total_allocated_, other.total_allocated_);
|
|
909
1678
|
return *this;
|
|
910
1679
|
}
|
|
911
1680
|
|
|
@@ -930,6 +1699,7 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
930
1699
|
last_usage_ = head_size();
|
|
931
1700
|
last_capacity_ = min_capacity();
|
|
932
1701
|
wasted_space_ = 0;
|
|
1702
|
+
total_allocated_ = 0;
|
|
933
1703
|
}
|
|
934
1704
|
|
|
935
1705
|
/**
|
|
@@ -968,6 +1738,7 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
968
1738
|
last_arena_ = new_arena;
|
|
969
1739
|
last_capacity_ = new_cap;
|
|
970
1740
|
last_usage_ = head_size();
|
|
1741
|
+
total_allocated_ += new_cap;
|
|
971
1742
|
}
|
|
972
1743
|
|
|
973
1744
|
wasted_space_ += extended_bytes - count_bytes;
|
|
@@ -978,17 +1749,7 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
978
1749
|
* @brief Returns the amount of memory used by the allocator across all arenas.
|
|
979
1750
|
* @return The amount of space in bytes.
|
|
980
1751
|
*/
|
|
981
|
-
std::size_t total_allocated() const noexcept {
|
|
982
|
-
if (!last_arena_)
|
|
983
|
-
return 0;
|
|
984
|
-
std::size_t total_used = 0;
|
|
985
|
-
std::size_t last_capacity = last_capacity_;
|
|
986
|
-
do {
|
|
987
|
-
total_used += last_capacity;
|
|
988
|
-
last_capacity /= capacity_multiplier();
|
|
989
|
-
} while (last_capacity >= min_capacity());
|
|
990
|
-
return total_used;
|
|
991
|
-
}
|
|
1752
|
+
std::size_t total_allocated() const noexcept { return total_allocated_; }
|
|
992
1753
|
|
|
993
1754
|
/**
|
|
994
1755
|
* @brief Returns the amount of wasted space due to alignment.
|
|
@@ -1132,6 +1893,10 @@ template <> struct cast_gt<i8_t, i8_t> {
|
|
|
1132
1893
|
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
1133
1894
|
};
|
|
1134
1895
|
|
|
1896
|
+
template <> struct cast_gt<u8_t, u8_t> {
|
|
1897
|
+
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
1898
|
+
};
|
|
1899
|
+
|
|
1135
1900
|
template <> struct cast_gt<b1x8_t, b1x8_t> {
|
|
1136
1901
|
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
1137
1902
|
};
|
|
@@ -1200,6 +1965,31 @@ template <typename to_scalar_at> struct cast_from_i8_gt {
|
|
|
1200
1965
|
}
|
|
1201
1966
|
};
|
|
1202
1967
|
|
|
1968
|
+
template <typename from_scalar_at> struct cast_to_u8_gt {
|
|
1969
|
+
inline static bool try_(byte_t const* input, std::size_t dim, byte_t* output) noexcept {
|
|
1970
|
+
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
1971
|
+
std::uint8_t* typed_output = reinterpret_cast<std::uint8_t*>(output);
|
|
1972
|
+
double magnitude = 0.0;
|
|
1973
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
1974
|
+
magnitude += (double)typed_input[i] * (double)typed_input[i];
|
|
1975
|
+
magnitude = std::sqrt(magnitude);
|
|
1976
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
1977
|
+
typed_output[i] =
|
|
1978
|
+
static_cast<std::uint8_t>(usearch::clamp<double>(typed_input[i] * 255.0 / magnitude, 0.0, 255.0));
|
|
1979
|
+
return true;
|
|
1980
|
+
}
|
|
1981
|
+
};
|
|
1982
|
+
|
|
1983
|
+
template <typename to_scalar_at> struct cast_from_u8_gt {
|
|
1984
|
+
static bool try_(byte_t const* input, std::size_t dim, byte_t* output) noexcept {
|
|
1985
|
+
std::uint8_t const* typed_input = reinterpret_cast<std::uint8_t const*>(input);
|
|
1986
|
+
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
1987
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
1988
|
+
typed_output[i] = static_cast<to_scalar_at>(typed_input[i]) / 255.f;
|
|
1989
|
+
return true;
|
|
1990
|
+
}
|
|
1991
|
+
};
|
|
1992
|
+
|
|
1203
1993
|
template <> struct cast_gt<i8_t, f16_bits_t> : public cast_from_i8_gt<f16_t> {};
|
|
1204
1994
|
template <> struct cast_gt<i8_t, bf16_bits_t> : public cast_from_i8_gt<bf16_t> {};
|
|
1205
1995
|
template <> struct cast_gt<i8_t, f32_t> : public cast_from_i8_gt<f32_t> {};
|
|
@@ -1223,6 +2013,101 @@ template <> struct cast_gt<f64_t, b1x8_t> : public cast_to_b1x8_gt<f64_t> {};
|
|
|
1223
2013
|
template <> struct cast_gt<b1x8_t, i8_t> : public cast_from_b1x8_gt<i8_t> {};
|
|
1224
2014
|
template <> struct cast_gt<i8_t, b1x8_t> : public cast_to_b1x8_gt<i8_t> {};
|
|
1225
2015
|
|
|
2016
|
+
template <typename from_at, typename to_at> struct cast_through_f32_gt {
|
|
2017
|
+
static bool try_(byte_t const* input, std::size_t dim, byte_t* output) noexcept {
|
|
2018
|
+
from_at const* in = reinterpret_cast<from_at const*>(input);
|
|
2019
|
+
to_at* out = reinterpret_cast<to_at*>(output);
|
|
2020
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
2021
|
+
out[i] = to_at(float(in[i]));
|
|
2022
|
+
return true;
|
|
2023
|
+
}
|
|
2024
|
+
};
|
|
2025
|
+
|
|
2026
|
+
template <> struct cast_gt<e5m2_bits_t, e5m2_bits_t> {
|
|
2027
|
+
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
2028
|
+
};
|
|
2029
|
+
template <> struct cast_gt<e5m2_bits_t, f32_t> : public cast_through_f32_gt<e5m2_t, f32_t> {};
|
|
2030
|
+
template <> struct cast_gt<f32_t, e5m2_bits_t> : public cast_through_f32_gt<f32_t, e5m2_t> {};
|
|
2031
|
+
template <> struct cast_gt<e5m2_bits_t, f64_t> : public cast_through_f32_gt<e5m2_t, f64_t> {};
|
|
2032
|
+
template <> struct cast_gt<f64_t, e5m2_bits_t> : public cast_through_f32_gt<f64_t, e5m2_t> {};
|
|
2033
|
+
template <> struct cast_gt<e5m2_bits_t, f16_bits_t> : public cast_through_f32_gt<e5m2_t, f16_t> {};
|
|
2034
|
+
template <> struct cast_gt<f16_bits_t, e5m2_bits_t> : public cast_through_f32_gt<f16_t, e5m2_t> {};
|
|
2035
|
+
template <> struct cast_gt<e5m2_bits_t, bf16_bits_t> : public cast_through_f32_gt<e5m2_t, bf16_t> {};
|
|
2036
|
+
template <> struct cast_gt<bf16_bits_t, e5m2_bits_t> : public cast_through_f32_gt<bf16_t, e5m2_t> {};
|
|
2037
|
+
template <> struct cast_gt<e5m2_bits_t, i8_t> : public cast_to_i8_gt<e5m2_t> {};
|
|
2038
|
+
template <> struct cast_gt<i8_t, e5m2_bits_t> : public cast_from_i8_gt<e5m2_t> {};
|
|
2039
|
+
template <> struct cast_gt<e5m2_bits_t, b1x8_t> : public cast_to_b1x8_gt<e5m2_t> {};
|
|
2040
|
+
template <> struct cast_gt<b1x8_t, e5m2_bits_t> : public cast_from_b1x8_gt<e5m2_t> {};
|
|
2041
|
+
|
|
2042
|
+
template <> struct cast_gt<e4m3_bits_t, e4m3_bits_t> {
|
|
2043
|
+
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
2044
|
+
};
|
|
2045
|
+
template <> struct cast_gt<e4m3_bits_t, f32_t> : public cast_through_f32_gt<e4m3_t, f32_t> {};
|
|
2046
|
+
template <> struct cast_gt<f32_t, e4m3_bits_t> : public cast_through_f32_gt<f32_t, e4m3_t> {};
|
|
2047
|
+
template <> struct cast_gt<e4m3_bits_t, f64_t> : public cast_through_f32_gt<e4m3_t, f64_t> {};
|
|
2048
|
+
template <> struct cast_gt<f64_t, e4m3_bits_t> : public cast_through_f32_gt<f64_t, e4m3_t> {};
|
|
2049
|
+
template <> struct cast_gt<e4m3_bits_t, f16_bits_t> : public cast_through_f32_gt<e4m3_t, f16_t> {};
|
|
2050
|
+
template <> struct cast_gt<f16_bits_t, e4m3_bits_t> : public cast_through_f32_gt<f16_t, e4m3_t> {};
|
|
2051
|
+
template <> struct cast_gt<e4m3_bits_t, bf16_bits_t> : public cast_through_f32_gt<e4m3_t, bf16_t> {};
|
|
2052
|
+
template <> struct cast_gt<bf16_bits_t, e4m3_bits_t> : public cast_through_f32_gt<bf16_t, e4m3_t> {};
|
|
2053
|
+
template <> struct cast_gt<e4m3_bits_t, i8_t> : public cast_to_i8_gt<e4m3_t> {};
|
|
2054
|
+
template <> struct cast_gt<i8_t, e4m3_bits_t> : public cast_from_i8_gt<e4m3_t> {};
|
|
2055
|
+
template <> struct cast_gt<e4m3_bits_t, b1x8_t> : public cast_to_b1x8_gt<e4m3_t> {};
|
|
2056
|
+
template <> struct cast_gt<b1x8_t, e4m3_bits_t> : public cast_from_b1x8_gt<e4m3_t> {};
|
|
2057
|
+
|
|
2058
|
+
template <> struct cast_gt<e2m3_bits_t, e2m3_bits_t> {
|
|
2059
|
+
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
2060
|
+
};
|
|
2061
|
+
template <> struct cast_gt<e2m3_bits_t, f32_t> : public cast_through_f32_gt<e2m3_t, f32_t> {};
|
|
2062
|
+
template <> struct cast_gt<f32_t, e2m3_bits_t> : public cast_through_f32_gt<f32_t, e2m3_t> {};
|
|
2063
|
+
template <> struct cast_gt<e2m3_bits_t, f64_t> : public cast_through_f32_gt<e2m3_t, f64_t> {};
|
|
2064
|
+
template <> struct cast_gt<f64_t, e2m3_bits_t> : public cast_through_f32_gt<f64_t, e2m3_t> {};
|
|
2065
|
+
template <> struct cast_gt<e2m3_bits_t, f16_bits_t> : public cast_through_f32_gt<e2m3_t, f16_t> {};
|
|
2066
|
+
template <> struct cast_gt<f16_bits_t, e2m3_bits_t> : public cast_through_f32_gt<f16_t, e2m3_t> {};
|
|
2067
|
+
template <> struct cast_gt<e2m3_bits_t, bf16_bits_t> : public cast_through_f32_gt<e2m3_t, bf16_t> {};
|
|
2068
|
+
template <> struct cast_gt<bf16_bits_t, e2m3_bits_t> : public cast_through_f32_gt<bf16_t, e2m3_t> {};
|
|
2069
|
+
template <> struct cast_gt<e2m3_bits_t, i8_t> : public cast_to_i8_gt<e2m3_t> {};
|
|
2070
|
+
template <> struct cast_gt<i8_t, e2m3_bits_t> : public cast_from_i8_gt<e2m3_t> {};
|
|
2071
|
+
template <> struct cast_gt<e2m3_bits_t, b1x8_t> : public cast_to_b1x8_gt<e2m3_t> {};
|
|
2072
|
+
template <> struct cast_gt<b1x8_t, e2m3_bits_t> : public cast_from_b1x8_gt<e2m3_t> {};
|
|
2073
|
+
|
|
2074
|
+
template <> struct cast_gt<e3m2_bits_t, e3m2_bits_t> {
|
|
2075
|
+
static bool try_(byte_t const*, std::size_t, byte_t*) noexcept { return false; }
|
|
2076
|
+
};
|
|
2077
|
+
template <> struct cast_gt<e3m2_bits_t, f32_t> : public cast_through_f32_gt<e3m2_t, f32_t> {};
|
|
2078
|
+
template <> struct cast_gt<f32_t, e3m2_bits_t> : public cast_through_f32_gt<f32_t, e3m2_t> {};
|
|
2079
|
+
template <> struct cast_gt<e3m2_bits_t, f64_t> : public cast_through_f32_gt<e3m2_t, f64_t> {};
|
|
2080
|
+
template <> struct cast_gt<f64_t, e3m2_bits_t> : public cast_through_f32_gt<f64_t, e3m2_t> {};
|
|
2081
|
+
template <> struct cast_gt<e3m2_bits_t, f16_bits_t> : public cast_through_f32_gt<e3m2_t, f16_t> {};
|
|
2082
|
+
template <> struct cast_gt<f16_bits_t, e3m2_bits_t> : public cast_through_f32_gt<f16_t, e3m2_t> {};
|
|
2083
|
+
template <> struct cast_gt<e3m2_bits_t, bf16_bits_t> : public cast_through_f32_gt<e3m2_t, bf16_t> {};
|
|
2084
|
+
template <> struct cast_gt<bf16_bits_t, e3m2_bits_t> : public cast_through_f32_gt<bf16_t, e3m2_t> {};
|
|
2085
|
+
template <> struct cast_gt<e3m2_bits_t, i8_t> : public cast_to_i8_gt<e3m2_t> {};
|
|
2086
|
+
template <> struct cast_gt<i8_t, e3m2_bits_t> : public cast_from_i8_gt<e3m2_t> {};
|
|
2087
|
+
template <> struct cast_gt<e3m2_bits_t, b1x8_t> : public cast_to_b1x8_gt<e3m2_t> {};
|
|
2088
|
+
template <> struct cast_gt<b1x8_t, e3m2_bits_t> : public cast_from_b1x8_gt<e3m2_t> {};
|
|
2089
|
+
|
|
2090
|
+
template <> struct cast_gt<u8_t, f16_bits_t> : public cast_from_u8_gt<f16_t> {};
|
|
2091
|
+
template <> struct cast_gt<u8_t, bf16_bits_t> : public cast_from_u8_gt<bf16_t> {};
|
|
2092
|
+
template <> struct cast_gt<u8_t, f32_t> : public cast_from_u8_gt<f32_t> {};
|
|
2093
|
+
template <> struct cast_gt<u8_t, f64_t> : public cast_from_u8_gt<f64_t> {};
|
|
2094
|
+
template <> struct cast_gt<f16_bits_t, u8_t> : public cast_to_u8_gt<f16_t> {};
|
|
2095
|
+
template <> struct cast_gt<bf16_bits_t, u8_t> : public cast_to_u8_gt<bf16_t> {};
|
|
2096
|
+
template <> struct cast_gt<f32_t, u8_t> : public cast_to_u8_gt<f32_t> {};
|
|
2097
|
+
template <> struct cast_gt<f64_t, u8_t> : public cast_to_u8_gt<f64_t> {};
|
|
2098
|
+
template <> struct cast_gt<b1x8_t, u8_t> : public cast_from_b1x8_gt<u8_t> {};
|
|
2099
|
+
template <> struct cast_gt<u8_t, b1x8_t> : public cast_to_b1x8_gt<u8_t> {};
|
|
2100
|
+
template <> struct cast_gt<e5m2_bits_t, u8_t> : public cast_to_u8_gt<e5m2_t> {};
|
|
2101
|
+
template <> struct cast_gt<u8_t, e5m2_bits_t> : public cast_from_u8_gt<e5m2_t> {};
|
|
2102
|
+
template <> struct cast_gt<e4m3_bits_t, u8_t> : public cast_to_u8_gt<e4m3_t> {};
|
|
2103
|
+
template <> struct cast_gt<u8_t, e4m3_bits_t> : public cast_from_u8_gt<e4m3_t> {};
|
|
2104
|
+
template <> struct cast_gt<e2m3_bits_t, u8_t> : public cast_to_u8_gt<e2m3_t> {};
|
|
2105
|
+
template <> struct cast_gt<u8_t, e2m3_bits_t> : public cast_from_u8_gt<e2m3_t> {};
|
|
2106
|
+
template <> struct cast_gt<e3m2_bits_t, u8_t> : public cast_to_u8_gt<e3m2_t> {};
|
|
2107
|
+
template <> struct cast_gt<u8_t, e3m2_bits_t> : public cast_from_u8_gt<e3m2_t> {};
|
|
2108
|
+
template <> struct cast_gt<i8_t, u8_t> : public cast_to_u8_gt<i8_t> {};
|
|
2109
|
+
template <> struct cast_gt<u8_t, i8_t> : public cast_from_u8_gt<i8_t> {};
|
|
2110
|
+
|
|
1226
2111
|
/**
|
|
1227
2112
|
* @brief Type-punned array casting function.
|
|
1228
2113
|
* Arguments: input buffer, bytes in input buffer, output buffer.
|
|
@@ -1237,20 +2122,30 @@ using cast_punned_t = bool (*)(byte_t const*, std::size_t, byte_t*);
|
|
|
1237
2122
|
*/
|
|
1238
2123
|
struct casts_punned_t {
|
|
1239
2124
|
struct group_t {
|
|
1240
|
-
cast_punned_t b1x8{};
|
|
1241
|
-
cast_punned_t i8{};
|
|
1242
|
-
cast_punned_t f16{};
|
|
1243
|
-
cast_punned_t bf16{};
|
|
1244
|
-
cast_punned_t f32{};
|
|
1245
2125
|
cast_punned_t f64{};
|
|
2126
|
+
cast_punned_t f32{};
|
|
2127
|
+
cast_punned_t bf16{};
|
|
2128
|
+
cast_punned_t f16{};
|
|
2129
|
+
cast_punned_t e5m2{};
|
|
2130
|
+
cast_punned_t e4m3{};
|
|
2131
|
+
cast_punned_t e3m2{};
|
|
2132
|
+
cast_punned_t e2m3{};
|
|
2133
|
+
cast_punned_t i8{};
|
|
2134
|
+
cast_punned_t u8{};
|
|
2135
|
+
cast_punned_t b1x8{};
|
|
1246
2136
|
|
|
1247
2137
|
cast_punned_t operator[](scalar_kind_t scalar_kind) const noexcept {
|
|
1248
2138
|
switch (scalar_kind) {
|
|
1249
2139
|
case scalar_kind_t::f64_k: return f64;
|
|
1250
2140
|
case scalar_kind_t::f32_k: return f32;
|
|
1251
|
-
case scalar_kind_t::f16_k: return f16;
|
|
1252
2141
|
case scalar_kind_t::bf16_k: return bf16;
|
|
2142
|
+
case scalar_kind_t::f16_k: return f16;
|
|
2143
|
+
case scalar_kind_t::e5m2_k: return e5m2;
|
|
2144
|
+
case scalar_kind_t::e4m3_k: return e4m3;
|
|
2145
|
+
case scalar_kind_t::e3m2_k: return e3m2;
|
|
2146
|
+
case scalar_kind_t::e2m3_k: return e2m3;
|
|
1253
2147
|
case scalar_kind_t::i8_k: return i8;
|
|
2148
|
+
case scalar_kind_t::u8_k: return u8;
|
|
1254
2149
|
case scalar_kind_t::b1x8_k: return b1x8;
|
|
1255
2150
|
default: return nullptr;
|
|
1256
2151
|
}
|
|
@@ -1261,19 +2156,29 @@ struct casts_punned_t {
|
|
|
1261
2156
|
template <typename scalar_at> static casts_punned_t make() noexcept {
|
|
1262
2157
|
casts_punned_t result;
|
|
1263
2158
|
|
|
1264
|
-
result.from.b1x8 = &cast_gt<b1x8_t, scalar_at>::try_;
|
|
1265
|
-
result.from.i8 = &cast_gt<i8_t, scalar_at>::try_;
|
|
1266
|
-
result.from.f16 = &cast_gt<f16_t, scalar_at>::try_;
|
|
1267
|
-
result.from.bf16 = &cast_gt<bf16_t, scalar_at>::try_;
|
|
1268
|
-
result.from.f32 = &cast_gt<f32_t, scalar_at>::try_;
|
|
1269
2159
|
result.from.f64 = &cast_gt<f64_t, scalar_at>::try_;
|
|
2160
|
+
result.from.f32 = &cast_gt<f32_t, scalar_at>::try_;
|
|
2161
|
+
result.from.bf16 = &cast_gt<bf16_t, scalar_at>::try_;
|
|
2162
|
+
result.from.f16 = &cast_gt<f16_t, scalar_at>::try_;
|
|
2163
|
+
result.from.e5m2 = &cast_gt<e5m2_t, scalar_at>::try_;
|
|
2164
|
+
result.from.e4m3 = &cast_gt<e4m3_t, scalar_at>::try_;
|
|
2165
|
+
result.from.e3m2 = &cast_gt<e3m2_t, scalar_at>::try_;
|
|
2166
|
+
result.from.e2m3 = &cast_gt<e2m3_t, scalar_at>::try_;
|
|
2167
|
+
result.from.i8 = &cast_gt<i8_t, scalar_at>::try_;
|
|
2168
|
+
result.from.u8 = &cast_gt<u8_t, scalar_at>::try_;
|
|
2169
|
+
result.from.b1x8 = &cast_gt<b1x8_t, scalar_at>::try_;
|
|
1270
2170
|
|
|
1271
|
-
result.to.b1x8 = &cast_gt<scalar_at, b1x8_t>::try_;
|
|
1272
|
-
result.to.i8 = &cast_gt<scalar_at, i8_t>::try_;
|
|
1273
|
-
result.to.f16 = &cast_gt<scalar_at, f16_t>::try_;
|
|
1274
|
-
result.to.bf16 = &cast_gt<scalar_at, bf16_t>::try_;
|
|
1275
|
-
result.to.f32 = &cast_gt<scalar_at, f32_t>::try_;
|
|
1276
2171
|
result.to.f64 = &cast_gt<scalar_at, f64_t>::try_;
|
|
2172
|
+
result.to.f32 = &cast_gt<scalar_at, f32_t>::try_;
|
|
2173
|
+
result.to.bf16 = &cast_gt<scalar_at, bf16_t>::try_;
|
|
2174
|
+
result.to.f16 = &cast_gt<scalar_at, f16_t>::try_;
|
|
2175
|
+
result.to.e5m2 = &cast_gt<scalar_at, e5m2_t>::try_;
|
|
2176
|
+
result.to.e4m3 = &cast_gt<scalar_at, e4m3_t>::try_;
|
|
2177
|
+
result.to.e3m2 = &cast_gt<scalar_at, e3m2_t>::try_;
|
|
2178
|
+
result.to.e2m3 = &cast_gt<scalar_at, e2m3_t>::try_;
|
|
2179
|
+
result.to.i8 = &cast_gt<scalar_at, i8_t>::try_;
|
|
2180
|
+
result.to.u8 = &cast_gt<scalar_at, u8_t>::try_;
|
|
2181
|
+
result.to.b1x8 = &cast_gt<scalar_at, b1x8_t>::try_;
|
|
1277
2182
|
|
|
1278
2183
|
return result;
|
|
1279
2184
|
}
|
|
@@ -1282,9 +2187,14 @@ struct casts_punned_t {
|
|
|
1282
2187
|
switch (scalar_kind) {
|
|
1283
2188
|
case scalar_kind_t::f64_k: return casts_punned_t::make<f64_t>();
|
|
1284
2189
|
case scalar_kind_t::f32_k: return casts_punned_t::make<f32_t>();
|
|
1285
|
-
case scalar_kind_t::f16_k: return casts_punned_t::make<f16_t>();
|
|
1286
2190
|
case scalar_kind_t::bf16_k: return casts_punned_t::make<bf16_t>();
|
|
2191
|
+
case scalar_kind_t::f16_k: return casts_punned_t::make<f16_t>();
|
|
2192
|
+
case scalar_kind_t::e5m2_k: return casts_punned_t::make<e5m2_t>();
|
|
2193
|
+
case scalar_kind_t::e4m3_k: return casts_punned_t::make<e4m3_t>();
|
|
2194
|
+
case scalar_kind_t::e3m2_k: return casts_punned_t::make<e3m2_t>();
|
|
2195
|
+
case scalar_kind_t::e2m3_k: return casts_punned_t::make<e2m3_t>();
|
|
1287
2196
|
case scalar_kind_t::i8_k: return casts_punned_t::make<i8_t>();
|
|
2197
|
+
case scalar_kind_t::u8_k: return casts_punned_t::make<u8_t>();
|
|
1288
2198
|
case scalar_kind_t::b1x8_k: return casts_punned_t::make<b1x8_t>();
|
|
1289
2199
|
default: return {};
|
|
1290
2200
|
}
|
|
@@ -1539,7 +2449,8 @@ template <typename scalar_at = float, typename result_at = float> struct metric_
|
|
|
1539
2449
|
b_sq_sum += bi * bi;
|
|
1540
2450
|
}
|
|
1541
2451
|
result_t denom = (dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum);
|
|
1542
|
-
|
|
2452
|
+
// could be negative
|
|
2453
|
+
if (denom <= 0)
|
|
1543
2454
|
return 0;
|
|
1544
2455
|
result_t corr = dim * ab_sum - a_sum * b_sum;
|
|
1545
2456
|
denom = std::sqrt(denom);
|
|
@@ -1629,6 +2540,57 @@ struct metric_l2sq_i8_t {
|
|
|
1629
2540
|
}
|
|
1630
2541
|
};
|
|
1631
2542
|
|
|
2543
|
+
/**
|
|
2544
|
+
* @brief Cosine (Angular) distance for unsigned 8-bit integers using 32-bit intermediates.
|
|
2545
|
+
*/
|
|
2546
|
+
struct metric_cos_u8_t {
|
|
2547
|
+
using scalar_t = u8_t;
|
|
2548
|
+
using result_t = f32_t;
|
|
2549
|
+
|
|
2550
|
+
inline result_t operator()(u8_t const* a, u8_t const* b, std::size_t dim) const noexcept {
|
|
2551
|
+
std::int64_t ab{}, a2{}, b2{};
|
|
2552
|
+
#if USEARCH_USE_OPENMP
|
|
2553
|
+
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
2554
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
2555
|
+
#pragma clang loop vectorize(enable)
|
|
2556
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
2557
|
+
#pragma GCC ivdep
|
|
2558
|
+
#endif
|
|
2559
|
+
for (std::size_t i = 0; i != dim; i++) {
|
|
2560
|
+
std::int32_t ai{a[i]};
|
|
2561
|
+
std::int32_t bi{b[i]};
|
|
2562
|
+
ab += ai * bi;
|
|
2563
|
+
a2 += square(ai);
|
|
2564
|
+
b2 += square(bi);
|
|
2565
|
+
}
|
|
2566
|
+
result_t a2f = std::sqrt(static_cast<result_t>(a2));
|
|
2567
|
+
result_t b2f = std::sqrt(static_cast<result_t>(b2));
|
|
2568
|
+
return (ab != 0) ? (1.f - ab / (a2f * b2f)) : 0;
|
|
2569
|
+
}
|
|
2570
|
+
};
|
|
2571
|
+
|
|
2572
|
+
/**
|
|
2573
|
+
* @brief Squared Euclidean (L2) distance for unsigned 8-bit integers using 32-bit intermediates.
|
|
2574
|
+
*/
|
|
2575
|
+
struct metric_l2sq_u8_t {
|
|
2576
|
+
using scalar_t = u8_t;
|
|
2577
|
+
using result_t = f32_t;
|
|
2578
|
+
|
|
2579
|
+
inline result_t operator()(u8_t const* a, u8_t const* b, std::size_t dim) const noexcept {
|
|
2580
|
+
std::int32_t ab_deltas_sq{};
|
|
2581
|
+
#if USEARCH_USE_OPENMP
|
|
2582
|
+
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
2583
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
2584
|
+
#pragma clang loop vectorize(enable)
|
|
2585
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
2586
|
+
#pragma GCC ivdep
|
|
2587
|
+
#endif
|
|
2588
|
+
for (std::size_t i = 0; i != dim; i++)
|
|
2589
|
+
ab_deltas_sq += square(std::int32_t(a[i]) - std::int32_t(b[i]));
|
|
2590
|
+
return static_cast<result_t>(ab_deltas_sq);
|
|
2591
|
+
}
|
|
2592
|
+
};
|
|
2593
|
+
|
|
1632
2594
|
/**
|
|
1633
2595
|
* @brief Haversine distance for the shortest distance between two nodes on
|
|
1634
2596
|
* the surface of a 3D sphere, defined with latitude and longitude.
|
|
@@ -1670,10 +2632,193 @@ enum class metric_punned_signature_t {
|
|
|
1670
2632
|
array_array_state_k,
|
|
1671
2633
|
};
|
|
1672
2634
|
|
|
2635
|
+
#if USEARCH_USE_NUMKONG
|
|
2636
|
+
|
|
2637
|
+
/**
|
|
2638
|
+
* @brief Returns the result of `nk_capabilities()`, cached in a function-local static.
|
|
2639
|
+
* Every call-site that needs the current CPU's capability mask should use this
|
|
2640
|
+
* instead of caching the value in its own `static` local.
|
|
2641
|
+
*/
|
|
2642
|
+
inline nk_capability_t nk_cached_capabilities() noexcept {
|
|
2643
|
+
static nk_capability_t caps = [] {
|
|
2644
|
+
nk_capability_t c = nk_capabilities();
|
|
2645
|
+
nk_configure_thread(c);
|
|
2646
|
+
return c;
|
|
2647
|
+
}();
|
|
2648
|
+
return caps;
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
/**
|
|
2652
|
+
* @brief Converts a USearch `scalar_kind_t` to the corresponding NumKong `nk_dtype_t`.
|
|
2653
|
+
* @return The matching dtype, or `(nk_dtype_t)0` when there is no NumKong equivalent.
|
|
2654
|
+
*/
|
|
2655
|
+
inline nk_dtype_t scalar_kind_to_nk_dtype(scalar_kind_t sk) noexcept {
|
|
2656
|
+
switch (sk) {
|
|
2657
|
+
case scalar_kind_t::f64_k: return (nk_dtype_t)nk_f64_k;
|
|
2658
|
+
case scalar_kind_t::f32_k: return (nk_dtype_t)nk_f32_k;
|
|
2659
|
+
case scalar_kind_t::bf16_k: return (nk_dtype_t)nk_bf16_k;
|
|
2660
|
+
case scalar_kind_t::f16_k: return (nk_dtype_t)nk_f16_k;
|
|
2661
|
+
case scalar_kind_t::e5m2_k: return (nk_dtype_t)nk_e5m2_k;
|
|
2662
|
+
case scalar_kind_t::e4m3_k: return (nk_dtype_t)nk_e4m3_k;
|
|
2663
|
+
case scalar_kind_t::e3m2_k: return (nk_dtype_t)nk_e3m2_k;
|
|
2664
|
+
case scalar_kind_t::e2m3_k: return (nk_dtype_t)nk_e2m3_k;
|
|
2665
|
+
case scalar_kind_t::i8_k: return (nk_dtype_t)nk_i8_k;
|
|
2666
|
+
case scalar_kind_t::u8_k: return (nk_dtype_t)nk_u8_k;
|
|
2667
|
+
case scalar_kind_t::b1x8_k: return (nk_dtype_t)nk_u1_k;
|
|
2668
|
+
default: return (nk_dtype_t)0;
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
/**
|
|
2673
|
+
* @brief One entry in the ISA capability-to-name mapping table.
|
|
2674
|
+
*/
|
|
2675
|
+
struct isa_target_t {
|
|
2676
|
+
nk_capability_t cap;
|
|
2677
|
+
char const* name;
|
|
2678
|
+
};
|
|
2679
|
+
|
|
2680
|
+
/**
|
|
2681
|
+
* @brief Returns the static table mapping each NumKong capability bit to its human-readable name.
|
|
2682
|
+
* Both `hardware_acceleration_available` and `hardware_acceleration_compiled` use this table
|
|
2683
|
+
* together with the bitmasks from `nk_capabilities_available()` / `nk_capabilities_compiled()`.
|
|
2684
|
+
*/
|
|
2685
|
+
inline span_gt<isa_target_t const> isa_targets() noexcept {
|
|
2686
|
+
static isa_target_t const table[] = {
|
|
2687
|
+
{nk_cap_serial_k, "serial"},
|
|
2688
|
+
// x86
|
|
2689
|
+
{nk_cap_haswell_k, "haswell"},
|
|
2690
|
+
{nk_cap_skylake_k, "skylake"},
|
|
2691
|
+
{nk_cap_icelake_k, "icelake"},
|
|
2692
|
+
{nk_cap_genoa_k, "genoa"},
|
|
2693
|
+
{nk_cap_sapphire_k, "sapphire"},
|
|
2694
|
+
{nk_cap_sapphireamx_k, "sapphireamx"},
|
|
2695
|
+
{nk_cap_graniteamx_k, "graniteamx"},
|
|
2696
|
+
{nk_cap_turin_k, "turin"},
|
|
2697
|
+
{nk_cap_sierra_k, "sierra"},
|
|
2698
|
+
{nk_cap_alder_k, "alder"},
|
|
2699
|
+
{nk_cap_diamond_k, "diamond"},
|
|
2700
|
+
// ARM NEON
|
|
2701
|
+
{nk_cap_neon_k, "neon"},
|
|
2702
|
+
{nk_cap_neonhalf_k, "neonhalf"},
|
|
2703
|
+
{nk_cap_neonsdot_k, "neonsdot"},
|
|
2704
|
+
{nk_cap_neonbfdot_k, "neonbfdot"},
|
|
2705
|
+
{nk_cap_neonfhm_k, "neonfhm"},
|
|
2706
|
+
{nk_cap_neonfp8_k, "neonfp8"},
|
|
2707
|
+
// ARM SVE
|
|
2708
|
+
{nk_cap_sve_k, "sve"},
|
|
2709
|
+
{nk_cap_svehalf_k, "svehalf"},
|
|
2710
|
+
{nk_cap_svesdot_k, "svesdot"},
|
|
2711
|
+
{nk_cap_svebfdot_k, "svebfdot"},
|
|
2712
|
+
{nk_cap_sve2_k, "sve2"},
|
|
2713
|
+
{nk_cap_sve2p1_k, "sve2p1"},
|
|
2714
|
+
// ARM SME
|
|
2715
|
+
{nk_cap_sme_k, "sme"},
|
|
2716
|
+
{nk_cap_sme2_k, "sme2"},
|
|
2717
|
+
{nk_cap_sme2p1_k, "sme2p1"},
|
|
2718
|
+
{nk_cap_smef64_k, "smef64"},
|
|
2719
|
+
{nk_cap_smefa64_k, "smefa64"},
|
|
2720
|
+
{nk_cap_smehalf_k, "smehalf"},
|
|
2721
|
+
{nk_cap_smebf16_k, "smebf16"},
|
|
2722
|
+
{nk_cap_smelut2_k, "smelut2"},
|
|
2723
|
+
{nk_cap_smebi32_k, "smebi32"},
|
|
2724
|
+
// RISC-V
|
|
2725
|
+
{nk_cap_rvv_k, "rvv"},
|
|
2726
|
+
{nk_cap_rvvhalf_k, "rvvhalf"},
|
|
2727
|
+
{nk_cap_rvvbf16_k, "rvvbf16"},
|
|
2728
|
+
{nk_cap_rvvbb_k, "rvvbb"},
|
|
2729
|
+
// WebAssembly
|
|
2730
|
+
{nk_cap_v128relaxed_k, "v128relaxed"},
|
|
2731
|
+
// LoongArch
|
|
2732
|
+
{nk_cap_loongsonasx_k, "loongsonasx"},
|
|
2733
|
+
// IBM Power
|
|
2734
|
+
{nk_cap_powervsx_k, "powervsx"},
|
|
2735
|
+
};
|
|
2736
|
+
return {table, sizeof(table) / sizeof(table[0])};
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
/**
|
|
2740
|
+
* @brief Returns the human-readable name for a single capability bit.
|
|
2741
|
+
* @param cap A single `nk_capability_t` bit (not a bitmask of multiple capabilities).
|
|
2742
|
+
* @return The name string, or "unknown" if the bit is not in the table.
|
|
2743
|
+
*/
|
|
2744
|
+
inline char const* capability_name(nk_capability_t cap) noexcept {
|
|
2745
|
+
span_gt<isa_target_t const> table = isa_targets();
|
|
2746
|
+
for (std::size_t i = 0; i != table.size(); ++i)
|
|
2747
|
+
if (table[i].cap == cap)
|
|
2748
|
+
return table[i].name;
|
|
2749
|
+
return "unknown";
|
|
2750
|
+
}
|
|
2751
|
+
|
|
2752
|
+
/**
|
|
2753
|
+
* @brief Formats all ISA targets whose bits are set in `caps` into a comma-separated string.
|
|
2754
|
+
* @param caps Capability bitmask to filter against.
|
|
2755
|
+
* @param buf Output buffer.
|
|
2756
|
+
* @param buf_len Size of the output buffer in bytes.
|
|
2757
|
+
* @return The number of characters written (excluding the null terminator).
|
|
2758
|
+
*/
|
|
2759
|
+
inline std::size_t isa_targets_format(nk_capability_t caps, char* buf, std::size_t buf_len) noexcept {
|
|
2760
|
+
if (!buf_len)
|
|
2761
|
+
return 0;
|
|
2762
|
+
span_gt<isa_target_t const> table = isa_targets();
|
|
2763
|
+
char* p = buf;
|
|
2764
|
+
char* end = buf + buf_len - 1;
|
|
2765
|
+
std::size_t matched = 0;
|
|
2766
|
+
for (std::size_t i = 0; i != table.size(); ++i) {
|
|
2767
|
+
if (!(caps & table[i].cap))
|
|
2768
|
+
continue;
|
|
2769
|
+
std::size_t name_len = std::strlen(table[i].name);
|
|
2770
|
+
std::size_t needed = name_len + (matched ? 2 : 0);
|
|
2771
|
+
if (p + needed > end)
|
|
2772
|
+
break;
|
|
2773
|
+
if (matched++)
|
|
2774
|
+
*p++ = ',', *p++ = ' ';
|
|
2775
|
+
std::memcpy(p, table[i].name, name_len);
|
|
2776
|
+
p += name_len;
|
|
2777
|
+
}
|
|
2778
|
+
*p = '\0';
|
|
2779
|
+
return static_cast<std::size_t>(p - buf);
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
/**
|
|
2783
|
+
* @brief Returns a comma-separated list of ISAs available at runtime (compiled AND supported by CPU).
|
|
2784
|
+
* @return Pointer to a static null-terminated string. Thread-safe after first call.
|
|
2785
|
+
*/
|
|
2786
|
+
inline char const* hardware_acceleration_available() noexcept {
|
|
2787
|
+
static char buf[1024];
|
|
2788
|
+
static bool initialized = false;
|
|
2789
|
+
if (!initialized) {
|
|
2790
|
+
nk_cached_capabilities(); // ensures nk_configure_thread is called
|
|
2791
|
+
isa_targets_format(nk_capabilities_available(), buf, sizeof(buf));
|
|
2792
|
+
initialized = true;
|
|
2793
|
+
}
|
|
2794
|
+
return buf;
|
|
2795
|
+
}
|
|
2796
|
+
|
|
2797
|
+
/**
|
|
2798
|
+
* @brief Returns a comma-separated list of ISAs that were compiled into this binary.
|
|
2799
|
+
* @return Pointer to a static null-terminated string. Thread-safe after first call.
|
|
2800
|
+
*/
|
|
2801
|
+
inline char const* hardware_acceleration_compiled() noexcept {
|
|
2802
|
+
static char buf[1024];
|
|
2803
|
+
static bool initialized = false;
|
|
2804
|
+
if (!initialized) {
|
|
2805
|
+
isa_targets_format(nk_capabilities_compiled(), buf, sizeof(buf));
|
|
2806
|
+
initialized = true;
|
|
2807
|
+
}
|
|
2808
|
+
return buf;
|
|
2809
|
+
}
|
|
2810
|
+
|
|
2811
|
+
#else
|
|
2812
|
+
|
|
2813
|
+
inline char const* hardware_acceleration_available() noexcept { return "serial"; }
|
|
2814
|
+
inline char const* hardware_acceleration_compiled() noexcept { return "serial"; }
|
|
2815
|
+
|
|
2816
|
+
#endif
|
|
2817
|
+
|
|
1673
2818
|
/**
|
|
1674
2819
|
* @brief Type-punned metric class, which unlike STL's `std::function` avoids any memory allocations.
|
|
1675
2820
|
* It also provides additional APIs to check, if SIMD hardware-acceleration is available.
|
|
1676
|
-
* Wraps the `
|
|
2821
|
+
* Wraps the `nk_metric_dense_punned_t` when available. The auto-vectorized backend otherwise.
|
|
1677
2822
|
*/
|
|
1678
2823
|
class metric_punned_t {
|
|
1679
2824
|
public:
|
|
@@ -1700,8 +2845,8 @@ class metric_punned_t {
|
|
|
1700
2845
|
metric_kind_t metric_kind_ = metric_kind_t::unknown_k;
|
|
1701
2846
|
scalar_kind_t scalar_kind_ = scalar_kind_t::unknown_k;
|
|
1702
2847
|
|
|
1703
|
-
#if
|
|
1704
|
-
|
|
2848
|
+
#if USEARCH_USE_NUMKONG
|
|
2849
|
+
nk_capability_t isa_kind_ = nk_cap_serial_k;
|
|
1705
2850
|
#endif
|
|
1706
2851
|
|
|
1707
2852
|
public:
|
|
@@ -1728,7 +2873,7 @@ class metric_punned_t {
|
|
|
1728
2873
|
|
|
1729
2874
|
/**
|
|
1730
2875
|
* @brief Creates a metric of a natively supported kind, choosing the best
|
|
1731
|
-
* available backend internally or from
|
|
2876
|
+
* available backend internally or from NumKong.
|
|
1732
2877
|
*
|
|
1733
2878
|
* @param dimensions The number of elements in the input arrays.
|
|
1734
2879
|
* @param metric_kind The kind of metric to use.
|
|
@@ -1746,12 +2891,8 @@ class metric_punned_t {
|
|
|
1746
2891
|
metric.metric_kind_ = metric_kind;
|
|
1747
2892
|
metric.scalar_kind_ = scalar_kind;
|
|
1748
2893
|
|
|
1749
|
-
|
|
1750
|
-
if (!metric.configure_with_simsimd())
|
|
2894
|
+
if (!metric.configure_with_numkong())
|
|
1751
2895
|
metric.configure_with_autovec();
|
|
1752
|
-
#else
|
|
1753
|
-
metric.configure_with_autovec();
|
|
1754
|
-
#endif
|
|
1755
2896
|
|
|
1756
2897
|
return metric;
|
|
1757
2898
|
}
|
|
@@ -1827,27 +2968,11 @@ class metric_punned_t {
|
|
|
1827
2968
|
inline char const* isa_name() const noexcept {
|
|
1828
2969
|
if (!*this)
|
|
1829
2970
|
return "uninitialized";
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
case simsimd_cap_serial_k: return "serial";
|
|
1834
|
-
case simsimd_cap_neon_k: return "neon";
|
|
1835
|
-
case simsimd_cap_neon_i8_k: return "neon_i8";
|
|
1836
|
-
case simsimd_cap_neon_f16_k: return "neon_f16";
|
|
1837
|
-
case simsimd_cap_neon_bf16_k: return "neon_bf16";
|
|
1838
|
-
case simsimd_cap_sve_k: return "sve";
|
|
1839
|
-
case simsimd_cap_sve_i8_k: return "sve_i8";
|
|
1840
|
-
case simsimd_cap_sve_f16_k: return "sve_f16";
|
|
1841
|
-
case simsimd_cap_sve_bf16_k: return "sve_bf16";
|
|
1842
|
-
case simsimd_cap_haswell_k: return "haswell";
|
|
1843
|
-
case simsimd_cap_skylake_k: return "skylake";
|
|
1844
|
-
case simsimd_cap_ice_k: return "ice";
|
|
1845
|
-
case simsimd_cap_genoa_k: return "genoa";
|
|
1846
|
-
case simsimd_cap_sapphire_k: return "sapphire";
|
|
1847
|
-
default: return "unknown";
|
|
1848
|
-
}
|
|
1849
|
-
#endif
|
|
2971
|
+
#if USEARCH_USE_NUMKONG
|
|
2972
|
+
return capability_name(isa_kind_);
|
|
2973
|
+
#else
|
|
1850
2974
|
return "serial";
|
|
2975
|
+
#endif
|
|
1851
2976
|
}
|
|
1852
2977
|
|
|
1853
2978
|
inline std::size_t bytes_per_vector() const noexcept {
|
|
@@ -1859,63 +2984,83 @@ class metric_punned_t {
|
|
|
1859
2984
|
}
|
|
1860
2985
|
|
|
1861
2986
|
private:
|
|
1862
|
-
#if
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
2987
|
+
#if USEARCH_USE_NUMKONG
|
|
2988
|
+
/**
|
|
2989
|
+
* @brief Typed invoke template for NumKong kernels.
|
|
2990
|
+
*
|
|
2991
|
+
* The accumulator type and IP-reversal flag are selected once in `configure_with_numkong`
|
|
2992
|
+
* and baked into the member-function pointer stored in `metric_routed_`.
|
|
2993
|
+
*/
|
|
2994
|
+
template <typename accumulator_at, bool reverse_ak>
|
|
2995
|
+
#if defined(USEARCH_DEFINED_CLANG) || defined(USEARCH_DEFINED_GCC)
|
|
2996
|
+
__attribute__((no_sanitize("all")))
|
|
2997
|
+
#endif
|
|
2998
|
+
result_t invoke_numkong(uptr_t a, uptr_t b) const noexcept {
|
|
2999
|
+
accumulator_at result = 0;
|
|
3000
|
+
auto function_pointer = (nk_metric_dense_punned_t)(metric_ptr_);
|
|
3001
|
+
function_pointer(reinterpret_cast<void const*>(a), reinterpret_cast<void const*>(b), metric_third_arg_,
|
|
3002
|
+
&result);
|
|
3003
|
+
return reverse_ak ? (result_t)(1 - (result_t)result) : (result_t)result;
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
/// Shorthand: casts an `invoke_numkong` instantiation to the opaque `metric_routed_t` type.
|
|
3007
|
+
template <typename accumulator_at, bool reverse_ak> //
|
|
3008
|
+
static metric_routed_t numkong_routed() noexcept {
|
|
3009
|
+
return reinterpret_cast<metric_routed_t>(&metric_punned_t::invoke_numkong<accumulator_at, reverse_ak>);
|
|
3010
|
+
}
|
|
3011
|
+
|
|
3012
|
+
bool configure_with_numkong(nk_capability_t simd_caps) noexcept {
|
|
3013
|
+
nk_kernel_kind_t kind = (nk_kernel_kind_t)0;
|
|
1867
3014
|
switch (metric_kind_) {
|
|
1868
|
-
case metric_kind_t::ip_k: kind =
|
|
1869
|
-
case metric_kind_t::cos_k: kind =
|
|
1870
|
-
case metric_kind_t::l2sq_k: kind =
|
|
1871
|
-
case metric_kind_t::hamming_k: kind =
|
|
1872
|
-
case metric_kind_t::tanimoto_k: kind =
|
|
1873
|
-
case metric_kind_t::jaccard_k: kind =
|
|
1874
|
-
default:
|
|
1875
|
-
}
|
|
1876
|
-
switch (scalar_kind_) {
|
|
1877
|
-
case scalar_kind_t::f32_k: datatype = simsimd_datatype_f32_k; break;
|
|
1878
|
-
case scalar_kind_t::f64_k: datatype = simsimd_datatype_f64_k; break;
|
|
1879
|
-
case scalar_kind_t::f16_k: datatype = simsimd_datatype_f16_k; break;
|
|
1880
|
-
case scalar_kind_t::bf16_k: datatype = simsimd_datatype_bf16_k; break;
|
|
1881
|
-
case scalar_kind_t::i8_k: datatype = simsimd_datatype_i8_k; break;
|
|
1882
|
-
case scalar_kind_t::b1x8_k: datatype = simsimd_datatype_b8_k; break;
|
|
1883
|
-
default: break;
|
|
3015
|
+
case metric_kind_t::ip_k: kind = nk_kernel_dot_k; break;
|
|
3016
|
+
case metric_kind_t::cos_k: kind = nk_kernel_angular_k; break;
|
|
3017
|
+
case metric_kind_t::l2sq_k: kind = nk_kernel_sqeuclidean_k; break;
|
|
3018
|
+
case metric_kind_t::hamming_k: kind = nk_kernel_hamming_k; break;
|
|
3019
|
+
case metric_kind_t::tanimoto_k: kind = nk_kernel_jaccard_k; break;
|
|
3020
|
+
case metric_kind_t::jaccard_k: kind = nk_kernel_jaccard_k; break;
|
|
3021
|
+
default: return false;
|
|
1884
3022
|
}
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
3023
|
+
nk_dtype_t datatype = scalar_kind_to_nk_dtype(scalar_kind_);
|
|
3024
|
+
nk_metric_dense_punned_t simd_metric = NULL;
|
|
3025
|
+
nk_capability_t simd_kind = nk_cap_any_k;
|
|
3026
|
+
nk_find_kernel_punned(kind, datatype, simd_caps, (nk_kernel_punned_t*)&simd_metric, &simd_kind);
|
|
1889
3027
|
if (simd_metric == nullptr)
|
|
1890
3028
|
return false;
|
|
1891
3029
|
|
|
1892
3030
|
std::memcpy(&metric_ptr_, &simd_metric, sizeof(simd_metric));
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
3031
|
+
|
|
3032
|
+
// Select the typed invoke variant based on the kernel's output dtype to avoid per-call branching.
|
|
3033
|
+
// The output type depends on both scalar_kind and metric_kind (e.g. i8 dot->i32, i8 l2sq->u32, i8 cos->f32).
|
|
3034
|
+
nk_dtype_t out_dtype = nk_kernel_output_dtype(kind, datatype);
|
|
3035
|
+
bool is_ip = (metric_kind_ == metric_kind_t::ip_k);
|
|
3036
|
+
switch (out_dtype) {
|
|
3037
|
+
case nk_f64_k:
|
|
3038
|
+
metric_routed_ = is_ip ? numkong_routed<nk_f64_t, true>() : numkong_routed<nk_f64_t, false>();
|
|
3039
|
+
break;
|
|
3040
|
+
case nk_f32_k:
|
|
3041
|
+
metric_routed_ = is_ip ? numkong_routed<nk_f32_t, true>() : numkong_routed<nk_f32_t, false>();
|
|
3042
|
+
break;
|
|
3043
|
+
case nk_i32_k:
|
|
3044
|
+
metric_routed_ = is_ip ? numkong_routed<std::int32_t, true>() : numkong_routed<std::int32_t, false>();
|
|
3045
|
+
break;
|
|
3046
|
+
case nk_u32_k:
|
|
3047
|
+
metric_routed_ = is_ip ? numkong_routed<std::uint32_t, true>() : numkong_routed<std::uint32_t, false>();
|
|
3048
|
+
break;
|
|
3049
|
+
default: metric_routed_ = is_ip ? numkong_routed<nk_f64_t, true>() : numkong_routed<nk_f64_t, false>(); break;
|
|
3050
|
+
}
|
|
1896
3051
|
isa_kind_ = simd_kind;
|
|
3052
|
+
|
|
3053
|
+
// NumKong binary-set kernels (Hamming, Jaccard/Tanimoto) expect the third argument
|
|
3054
|
+
// to be the number of dimensions (bits), not the number of bytes.
|
|
3055
|
+
if (scalar_kind_ == scalar_kind_t::b1x8_k)
|
|
3056
|
+
metric_third_arg_ = dimensions_;
|
|
3057
|
+
|
|
1897
3058
|
return true;
|
|
1898
3059
|
}
|
|
1899
|
-
bool configure_with_simsimd() noexcept {
|
|
1900
|
-
static simsimd_capability_t static_capabilities = simsimd_capabilities();
|
|
1901
|
-
return configure_with_simsimd(static_capabilities);
|
|
1902
|
-
}
|
|
1903
3060
|
|
|
1904
|
-
|
|
1905
|
-
__attribute__((no_sanitize("all")))
|
|
1906
|
-
#endif
|
|
1907
|
-
result_t
|
|
1908
|
-
invoke_simsimd(uptr_t a, uptr_t b) const noexcept {
|
|
1909
|
-
simsimd_distance_t result;
|
|
1910
|
-
// Here `reinterpret_cast` raises warning and UBSan reports an issue... we know what we are doing!
|
|
1911
|
-
auto function_pointer = (simsimd_metric_dense_punned_t)(metric_ptr_);
|
|
1912
|
-
function_pointer(reinterpret_cast<void const*>(a), reinterpret_cast<void const*>(b), metric_third_arg_,
|
|
1913
|
-
&result);
|
|
1914
|
-
return (result_t)result;
|
|
1915
|
-
}
|
|
1916
|
-
result_t invoke_simsimd_reverse(uptr_t a, uptr_t b) const noexcept { return 1 - invoke_simsimd(a, b); }
|
|
3061
|
+
bool configure_with_numkong() noexcept { return configure_with_numkong(nk_cached_capabilities()); }
|
|
1917
3062
|
#else
|
|
1918
|
-
bool
|
|
3063
|
+
bool configure_with_numkong() noexcept { return false; }
|
|
1919
3064
|
#endif
|
|
1920
3065
|
result_t invoke_array_array_third(uptr_t a, uptr_t b) const noexcept {
|
|
1921
3066
|
auto function_pointer = (metric_array_array_size_t)(metric_ptr_);
|
|
@@ -1931,70 +3076,108 @@ class metric_punned_t {
|
|
|
1931
3076
|
switch (metric_kind_) {
|
|
1932
3077
|
case metric_kind_t::ip_k: {
|
|
1933
3078
|
switch (scalar_kind_) {
|
|
3079
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f64_t>>; break;
|
|
3080
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f32_t>>; break;
|
|
1934
3081
|
case scalar_kind_t::bf16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<bf16_t, f32_t>>; break;
|
|
1935
|
-
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<i8_t, f32_t>>; break;
|
|
1936
3082
|
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f16_t, f32_t>>; break;
|
|
1937
|
-
case scalar_kind_t::
|
|
1938
|
-
case scalar_kind_t::
|
|
3083
|
+
case scalar_kind_t::e5m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<e5m2_t, f32_t>>; break;
|
|
3084
|
+
case scalar_kind_t::e4m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<e4m3_t, f32_t>>; break;
|
|
3085
|
+
case scalar_kind_t::e3m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<e3m2_t, f32_t>>; break;
|
|
3086
|
+
case scalar_kind_t::e2m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<e2m3_t, f32_t>>; break;
|
|
3087
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<i8_t, f32_t>>; break;
|
|
3088
|
+
case scalar_kind_t::u8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<u8_t, f32_t>>; break;
|
|
1939
3089
|
default: metric_ptr_ = 0; break;
|
|
1940
3090
|
}
|
|
1941
3091
|
break;
|
|
1942
3092
|
}
|
|
1943
3093
|
case metric_kind_t::cos_k: {
|
|
1944
3094
|
switch (scalar_kind_) {
|
|
3095
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f64_t>>; break;
|
|
3096
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f32_t>>; break;
|
|
1945
3097
|
case scalar_kind_t::bf16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<bf16_t, f32_t>>; break;
|
|
1946
|
-
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_i8_t>; break;
|
|
1947
3098
|
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f16_t, f32_t>>; break;
|
|
1948
|
-
case scalar_kind_t::
|
|
1949
|
-
case scalar_kind_t::
|
|
3099
|
+
case scalar_kind_t::e5m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<e5m2_t, f32_t>>; break;
|
|
3100
|
+
case scalar_kind_t::e4m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<e4m3_t, f32_t>>; break;
|
|
3101
|
+
case scalar_kind_t::e3m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<e3m2_t, f32_t>>; break;
|
|
3102
|
+
case scalar_kind_t::e2m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<e2m3_t, f32_t>>; break;
|
|
3103
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_i8_t>; break;
|
|
3104
|
+
case scalar_kind_t::u8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_u8_t>; break;
|
|
1950
3105
|
default: metric_ptr_ = 0; break;
|
|
1951
3106
|
}
|
|
1952
3107
|
break;
|
|
1953
3108
|
}
|
|
1954
3109
|
case metric_kind_t::l2sq_k: {
|
|
1955
3110
|
switch (scalar_kind_) {
|
|
3111
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f64_t>>; break;
|
|
3112
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f32_t>>; break;
|
|
1956
3113
|
case scalar_kind_t::bf16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<bf16_t, f32_t>>; break;
|
|
1957
|
-
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_i8_t>; break;
|
|
1958
3114
|
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f16_t, f32_t>>; break;
|
|
1959
|
-
case scalar_kind_t::
|
|
1960
|
-
case scalar_kind_t::
|
|
3115
|
+
case scalar_kind_t::e5m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<e5m2_t, f32_t>>; break;
|
|
3116
|
+
case scalar_kind_t::e4m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<e4m3_t, f32_t>>; break;
|
|
3117
|
+
case scalar_kind_t::e3m2_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<e3m2_t, f32_t>>; break;
|
|
3118
|
+
case scalar_kind_t::e2m3_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<e2m3_t, f32_t>>; break;
|
|
3119
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_i8_t>; break;
|
|
3120
|
+
case scalar_kind_t::u8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_u8_t>; break;
|
|
1961
3121
|
default: metric_ptr_ = 0; break;
|
|
1962
3122
|
}
|
|
1963
3123
|
break;
|
|
1964
3124
|
}
|
|
1965
3125
|
case metric_kind_t::pearson_k: {
|
|
1966
3126
|
switch (scalar_kind_) {
|
|
3127
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f64_t>>; break;
|
|
3128
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f32_t>>; break;
|
|
1967
3129
|
case scalar_kind_t::bf16_k:
|
|
1968
3130
|
metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<bf16_t, f32_t>>;
|
|
1969
3131
|
break;
|
|
1970
|
-
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<i8_t, f32_t>>; break;
|
|
1971
3132
|
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f16_t, f32_t>>; break;
|
|
1972
|
-
case scalar_kind_t::
|
|
1973
|
-
|
|
3133
|
+
case scalar_kind_t::e5m2_k:
|
|
3134
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<e5m2_t, f32_t>>;
|
|
3135
|
+
break;
|
|
3136
|
+
case scalar_kind_t::e4m3_k:
|
|
3137
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<e4m3_t, f32_t>>;
|
|
3138
|
+
break;
|
|
3139
|
+
case scalar_kind_t::e3m2_k:
|
|
3140
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<e3m2_t, f32_t>>;
|
|
3141
|
+
break;
|
|
3142
|
+
case scalar_kind_t::e2m3_k:
|
|
3143
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<e2m3_t, f32_t>>;
|
|
3144
|
+
break;
|
|
3145
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<i8_t, f32_t>>; break;
|
|
3146
|
+
case scalar_kind_t::u8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<u8_t, f32_t>>; break;
|
|
1974
3147
|
default: metric_ptr_ = 0; break;
|
|
1975
3148
|
}
|
|
1976
3149
|
break;
|
|
1977
3150
|
}
|
|
1978
3151
|
case metric_kind_t::haversine_k: {
|
|
1979
3152
|
switch (scalar_kind_) {
|
|
1980
|
-
case scalar_kind_t::bf16_k: metric_ptr_ = 0; break; //< Half-precision 2D vectors are silly.
|
|
1981
|
-
case scalar_kind_t::f16_k: metric_ptr_ = 0; break; //< Half-precision 2D vectors are silly.
|
|
1982
|
-
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f32_t>>; break;
|
|
1983
3153
|
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f64_t>>; break;
|
|
3154
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f32_t>>; break;
|
|
1984
3155
|
default: metric_ptr_ = 0; break;
|
|
1985
3156
|
}
|
|
1986
3157
|
break;
|
|
1987
3158
|
}
|
|
1988
3159
|
case metric_kind_t::divergence_k: {
|
|
1989
3160
|
switch (scalar_kind_) {
|
|
3161
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f64_t>>; break;
|
|
3162
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f32_t>>; break;
|
|
1990
3163
|
case scalar_kind_t::bf16_k:
|
|
1991
3164
|
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<bf16_t, f32_t>>;
|
|
1992
3165
|
break;
|
|
1993
3166
|
case scalar_kind_t::f16_k:
|
|
1994
3167
|
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f16_t, f32_t>>;
|
|
1995
3168
|
break;
|
|
1996
|
-
case scalar_kind_t::
|
|
1997
|
-
|
|
3169
|
+
case scalar_kind_t::e5m2_k:
|
|
3170
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<e5m2_t, f32_t>>;
|
|
3171
|
+
break;
|
|
3172
|
+
case scalar_kind_t::e4m3_k:
|
|
3173
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<e4m3_t, f32_t>>;
|
|
3174
|
+
break;
|
|
3175
|
+
case scalar_kind_t::e3m2_k:
|
|
3176
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<e3m2_t, f32_t>>;
|
|
3177
|
+
break;
|
|
3178
|
+
case scalar_kind_t::e2m3_k:
|
|
3179
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<e2m3_t, f32_t>>;
|
|
3180
|
+
break;
|
|
1998
3181
|
default: metric_ptr_ = 0; break;
|
|
1999
3182
|
}
|
|
2000
3183
|
break;
|
|
@@ -2084,10 +3267,9 @@ class exact_search_t {
|
|
|
2084
3267
|
std::size_t wanted, metric_punned_t const& metric, //
|
|
2085
3268
|
executor_at&& executor = executor_at{}, progress_at&& progress = progress_at{}) {
|
|
2086
3269
|
return operator()( //
|
|
2087
|
-
metric, //
|
|
2088
3270
|
reinterpret_cast<byte_t const*>(dataset.data()), dataset.size(), dataset.stride_bytes(), //
|
|
2089
3271
|
reinterpret_cast<byte_t const*>(queries.data()), queries.size(), queries.stride_bytes(), //
|
|
2090
|
-
wanted, executor, progress);
|
|
3272
|
+
wanted, metric, executor, progress);
|
|
2091
3273
|
}
|
|
2092
3274
|
|
|
2093
3275
|
template <typename executor_at = dummy_executor_t, typename progress_at = dummy_progress_t>
|
|
@@ -2097,9 +3279,8 @@ class exact_search_t {
|
|
|
2097
3279
|
std::size_t wanted, metric_punned_t const& metric, executor_at&& executor = executor_at{},
|
|
2098
3280
|
progress_at&& progress = progress_at{}) {
|
|
2099
3281
|
|
|
2100
|
-
// Allocate temporary memory to store the distance matrix
|
|
2101
|
-
//
|
|
2102
|
-
// In the new design we keep two buffers - original and transposed, as in-place transpositions
|
|
3282
|
+
// Allocate temporary memory to store the distance matrix.
|
|
3283
|
+
// We keep two buffers - original and transposed, as in-place transpositions
|
|
2103
3284
|
// of non-rectangular matrixes is expensive.
|
|
2104
3285
|
std::size_t tasks_count = dataset_count * queries_count;
|
|
2105
3286
|
if (keys_and_distances.size() < tasks_count * 2)
|
|
@@ -2145,9 +3326,6 @@ class exact_search_t {
|
|
|
2145
3326
|
executor.fixed(queries_count, [&](std::size_t, std::size_t query_idx) {
|
|
2146
3327
|
auto start = keys_and_distances_per_query + dataset_count * query_idx;
|
|
2147
3328
|
if (wanted > 1) {
|
|
2148
|
-
// TODO: Consider alternative sorting approaches
|
|
2149
|
-
// radix_sort(start, start + dataset_count, wanted);
|
|
2150
|
-
// std::sort(start, start + dataset_count, &smaller_distance);
|
|
2151
3329
|
std::partial_sort(start, start + wanted, start + dataset_count, &smaller_distance);
|
|
2152
3330
|
} else {
|
|
2153
3331
|
auto min_it = std::min_element(start, start + dataset_count, &smaller_distance);
|
|
@@ -2306,7 +3484,7 @@ template <typename allocator_at = std::allocator<char>> class kmeans_clustering_
|
|
|
2306
3484
|
return result.failed("No memory for result outputs!");
|
|
2307
3485
|
|
|
2308
3486
|
std::fill_n(point_to_centroid_index_buffer.data(), points_count, wanted_clusters);
|
|
2309
|
-
std::fill_n(point_to_centroid_distance_buffer.data(), points_count, std::numeric_limits<distance_t>::max());
|
|
3487
|
+
std::fill_n(point_to_centroid_distance_buffer.data(), points_count, (std::numeric_limits<distance_t>::max)());
|
|
2310
3488
|
|
|
2311
3489
|
// Initialize the casting kernel for quantization and export.
|
|
2312
3490
|
casts_punned_t casts = casts_punned_t::make(quantization_kind);
|
|
@@ -2352,7 +3530,7 @@ template <typename allocator_at = std::allocator<char>> class kmeans_clustering_
|
|
|
2352
3530
|
auto start_time = std::chrono::high_resolution_clock::now();
|
|
2353
3531
|
std::size_t iterations = 0;
|
|
2354
3532
|
std::size_t const min_points_shifted_per_iteration = static_cast<std::size_t>(min_shifts * points_count);
|
|
2355
|
-
f64_t last_aggregate_distance = std::numeric_limits<f64_t>::max();
|
|
3533
|
+
f64_t last_aggregate_distance = (std::numeric_limits<f64_t>::max)();
|
|
2356
3534
|
|
|
2357
3535
|
while (iterations < max_iterations) {
|
|
2358
3536
|
iterations++;
|
|
@@ -2363,7 +3541,7 @@ template <typename allocator_at = std::allocator<char>> class kmeans_clustering_
|
|
|
2363
3541
|
byte_t const* quantized_point =
|
|
2364
3542
|
points_quantized_buffer.data() + points_idx * stride_per_vector_quantized;
|
|
2365
3543
|
byte_t const* quantized_centroids = centroids_quantized_buffer.data();
|
|
2366
|
-
distance_t closest_distance_local = std::numeric_limits<distance_t>::max();
|
|
3544
|
+
distance_t closest_distance_local = (std::numeric_limits<distance_t>::max)();
|
|
2367
3545
|
std::size_t closest_idx_local = 0;
|
|
2368
3546
|
for (std::size_t centroid_idx = 0; centroid_idx < wanted_clusters; centroid_idx++) {
|
|
2369
3547
|
byte_t const* quantized_centroid = quantized_centroids + centroid_idx * stride_per_vector_quantized;
|
|
@@ -2737,12 +3915,17 @@ class flat_hash_multi_set_gt {
|
|
|
2737
3915
|
equals_t const& equals)
|
|
2738
3916
|
: index_(index), parent_(parent), query_(query), equals_(equals) {}
|
|
2739
3917
|
|
|
2740
|
-
// Pre-increment
|
|
3918
|
+
// Pre-increment: advance past tombstones and non-matching entries,
|
|
3919
|
+
// stopping at the next matching live entry or an empty slot.
|
|
2741
3920
|
equal_iterator_gt& operator++() {
|
|
2742
3921
|
do {
|
|
2743
3922
|
index_ = (index_ + 1) & (parent_->capacity_slots_ - 1);
|
|
2744
|
-
|
|
2745
|
-
|
|
3923
|
+
auto slot = parent_->slot_ref(index_);
|
|
3924
|
+
bool is_empty = ~slot.header.populated & slot.mask;
|
|
3925
|
+
bool is_match = !(slot.header.deleted & slot.mask) && equals_(slot.element, query_);
|
|
3926
|
+
if (is_empty || is_match)
|
|
3927
|
+
break;
|
|
3928
|
+
} while (true);
|
|
2746
3929
|
return *this;
|
|
2747
3930
|
}
|
|
2748
3931
|
|