usearch 2.21.4 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -24
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +7 -7
- package/c/lib.cpp +32 -14
- package/c/test.c +83 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +224 -48
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -1,2465 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file elementwise.h
|
|
3
|
-
* @brief SIMD-accelerated mixed-precision element-wise operations.
|
|
4
|
-
* @author Ash Vardanian
|
|
5
|
-
* @date October 16, 2024
|
|
6
|
-
*
|
|
7
|
-
* Contains following element-wise operations:
|
|
8
|
-
* - Sum (Add): R[i] = A[i] + B[i]
|
|
9
|
-
* - Scale (Multiply): R[i] = Alpha * A[i]
|
|
10
|
-
* - WSum or Weighted-Sum: R[i] = Alpha * A[i] + Beta * B[i]
|
|
11
|
-
* - FMA or Fused-Multiply-Add: R[i] = Alpha * A[i] * B[i] + Beta * C[i]
|
|
12
|
-
*
|
|
13
|
-
* This tiny set of operations if enough to implement a wide range of algorithms.
|
|
14
|
-
* To scale a vector by a scalar, just call WSum with $Beta$ = 0.
|
|
15
|
-
* To sum two vectors, just call WSum with $Alpha$ = $Beta$ = 1.
|
|
16
|
-
* To average two vectors, just call WSum with $Alpha$ = $Beta$ = 0.5.
|
|
17
|
-
* To multiply vectors element-wise, just call FMA with $Beta$ = 0.
|
|
18
|
-
*
|
|
19
|
-
* For datatypes:
|
|
20
|
-
* - 64-bit IEEE floating point numbers
|
|
21
|
-
* - 32-bit IEEE floating point numbers
|
|
22
|
-
* - 16-bit IEEE floating point numbers
|
|
23
|
-
* - 16-bit brain floating point numbers
|
|
24
|
-
* - 8-bit unsigned integers
|
|
25
|
-
* - 8-bit signed integers
|
|
26
|
-
*
|
|
27
|
-
* For hardware architectures:
|
|
28
|
-
* - Arm: NEON
|
|
29
|
-
* - x86: Haswell, Skylake, Sapphire
|
|
30
|
-
*
|
|
31
|
-
* We use `f16` for `i8` and `u8` arithmetic. This is because Arm received `f16` support earlier than `bf16`.
|
|
32
|
-
* For example, Apple M1 has `f16` support and `bf16` was only added in M2. On the other hand, on paper,
|
|
33
|
-
* AMD Genoa has `bf16` support, and `f16` is only available on Intel Sapphire Rapids and newer.
|
|
34
|
-
* Sadly, the SIMD support for `bf16` is limited to mixed-precision dot-products, which makes it useless here.
|
|
35
|
-
*
|
|
36
|
-
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
37
|
-
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
38
|
-
*/
|
|
39
|
-
#ifndef SIMSIMD_ELEMENTWISE_H
|
|
40
|
-
#define SIMSIMD_ELEMENTWISE_H
|
|
41
|
-
|
|
42
|
-
#include "types.h"
|
|
43
|
-
|
|
44
|
-
#ifdef __cplusplus
|
|
45
|
-
extern "C" {
|
|
46
|
-
#endif
|
|
47
|
-
|
|
48
|
-
/* Serial backends for all numeric types.
|
|
49
|
-
* By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
|
|
50
|
-
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
51
|
-
*/
|
|
52
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f64_serial( //
|
|
53
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
54
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
55
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_serial( //
|
|
56
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
57
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
58
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_serial( //
|
|
59
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
60
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
61
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_serial( //
|
|
62
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
63
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
64
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_serial( //
|
|
65
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
66
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
67
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_serial( //
|
|
68
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
69
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
70
|
-
|
|
71
|
-
SIMSIMD_PUBLIC void simsimd_fma_f64_serial( //
|
|
72
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
|
|
73
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
74
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_serial( //
|
|
75
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
|
|
76
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
77
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_serial( //
|
|
78
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
|
|
79
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
80
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_serial( //
|
|
81
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
|
|
82
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
83
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_serial( //
|
|
84
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
|
|
85
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
86
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_serial( //
|
|
87
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
|
|
88
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
89
|
-
|
|
90
|
-
#define SIMSIMD_MAKE_WSUM(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
|
|
91
|
-
SIMSIMD_PUBLIC void simsimd_wsum_##input_type##_##name( \
|
|
92
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
93
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
|
|
94
|
-
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
95
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
96
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
97
|
-
simsimd_##accumulator_type##_t ai_scaled = (simsimd_##accumulator_type##_t)(ai * alpha); \
|
|
98
|
-
simsimd_##accumulator_type##_t bi_scaled = (simsimd_##accumulator_type##_t)(bi * beta); \
|
|
99
|
-
simsimd_##accumulator_type##_t sum = ai_scaled + bi_scaled; \
|
|
100
|
-
convert_and_store(sum, result + i); \
|
|
101
|
-
} \
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
#define SIMSIMD_MAKE_FMA(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
|
|
105
|
-
SIMSIMD_PUBLIC void simsimd_fma_##input_type##_##name( \
|
|
106
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
|
|
107
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
|
|
108
|
-
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
109
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
110
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
111
|
-
simsimd_##accumulator_type##_t ci = load_and_convert(c + i); \
|
|
112
|
-
simsimd_##accumulator_type##_t abi_scaled = (simsimd_##accumulator_type##_t)(ai * bi * alpha); \
|
|
113
|
-
simsimd_##accumulator_type##_t ci_scaled = (simsimd_##accumulator_type##_t)(ci * beta); \
|
|
114
|
-
simsimd_##accumulator_type##_t sum = abi_scaled + ci_scaled; \
|
|
115
|
-
convert_and_store(sum, result + i); \
|
|
116
|
-
} \
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
SIMSIMD_MAKE_WSUM(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f64_serial
|
|
120
|
-
SIMSIMD_MAKE_WSUM(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_serial
|
|
121
|
-
SIMSIMD_MAKE_WSUM(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_serial
|
|
122
|
-
SIMSIMD_MAKE_WSUM(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_serial
|
|
123
|
-
SIMSIMD_MAKE_WSUM(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_wsum_i8_serial
|
|
124
|
-
SIMSIMD_MAKE_WSUM(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_wsum_u8_serial
|
|
125
|
-
|
|
126
|
-
SIMSIMD_MAKE_WSUM(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_accurate
|
|
127
|
-
SIMSIMD_MAKE_WSUM(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_accurate
|
|
128
|
-
SIMSIMD_MAKE_WSUM(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_accurate
|
|
129
|
-
SIMSIMD_MAKE_WSUM(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_wsum_i8_accurate
|
|
130
|
-
SIMSIMD_MAKE_WSUM(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_wsum_u8_accurate
|
|
131
|
-
|
|
132
|
-
SIMSIMD_MAKE_FMA(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f64_serial
|
|
133
|
-
SIMSIMD_MAKE_FMA(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_serial
|
|
134
|
-
SIMSIMD_MAKE_FMA(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_serial
|
|
135
|
-
SIMSIMD_MAKE_FMA(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_serial
|
|
136
|
-
SIMSIMD_MAKE_FMA(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_fma_i8_serial
|
|
137
|
-
SIMSIMD_MAKE_FMA(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_fma_u8_serial
|
|
138
|
-
|
|
139
|
-
SIMSIMD_MAKE_FMA(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_accurate
|
|
140
|
-
SIMSIMD_MAKE_FMA(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_accurate
|
|
141
|
-
SIMSIMD_MAKE_FMA(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_accurate
|
|
142
|
-
SIMSIMD_MAKE_FMA(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_fma_i8_accurate
|
|
143
|
-
SIMSIMD_MAKE_FMA(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_fma_u8_accurate
|
|
144
|
-
|
|
145
|
-
/* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
|
|
146
|
-
* By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
|
|
147
|
-
* server CPUs produced before 2023.
|
|
148
|
-
*/
|
|
149
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
|
|
150
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
151
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
152
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
|
|
153
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
154
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
155
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
|
|
156
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
157
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
158
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
|
|
159
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
160
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
161
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
|
|
162
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
163
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
164
|
-
|
|
165
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
|
|
166
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
|
|
167
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
168
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
|
|
169
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
|
|
170
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
171
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
|
|
172
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
|
|
173
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
174
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
|
|
175
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
|
|
176
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
177
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
|
|
178
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
|
|
179
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
180
|
-
|
|
181
|
-
/* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
|
|
182
|
-
* First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
|
|
183
|
-
* Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
|
|
184
|
-
* On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
|
|
185
|
-
* properly vectorized by recent compilers.
|
|
186
|
-
*/
|
|
187
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
|
|
188
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
189
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
190
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
|
|
191
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
192
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
193
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
|
|
194
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
195
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
196
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
|
|
197
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
198
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
199
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
|
|
200
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
201
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
202
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
|
|
203
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
204
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
205
|
-
SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
|
|
206
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
|
|
207
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
208
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
|
|
209
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
|
|
210
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
211
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
|
|
212
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
|
|
213
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
214
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
|
|
215
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
|
|
216
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
217
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
|
|
218
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
|
|
219
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
220
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
|
|
221
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
|
|
222
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
223
|
-
|
|
224
|
-
/* SIMD-powered backends for various generations of AVX512 CPUs.
|
|
225
|
-
* Unlike the distance metrics, the SIMD implementation of FMA and WSum benefits from aligned stores.
|
|
226
|
-
* Assuming the size of ZMM register matches the width of the cache line, we skip the unaligned head
|
|
227
|
-
* and tail of the output buffer, and only use aligned stores in the main loop.
|
|
228
|
-
*/
|
|
229
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
|
|
230
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
231
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
232
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
|
|
233
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
234
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
235
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
|
|
236
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
237
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
238
|
-
|
|
239
|
-
SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
|
|
240
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
|
|
241
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
|
|
242
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
|
|
243
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
|
|
244
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
|
|
245
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
|
|
246
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
|
|
247
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
|
|
248
|
-
|
|
249
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
|
|
250
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
251
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
252
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
|
|
253
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
254
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
255
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
|
|
256
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
257
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
258
|
-
|
|
259
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
|
|
260
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
|
|
261
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
|
|
262
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
|
|
263
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
|
|
264
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
|
|
265
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
|
|
266
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
|
|
267
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
|
|
268
|
-
|
|
269
|
-
#if _SIMSIMD_TARGET_X86
|
|
270
|
-
#if SIMSIMD_TARGET_HASWELL
|
|
271
|
-
#pragma GCC push_options
|
|
272
|
-
#pragma GCC target("avx2", "f16c", "fma")
|
|
273
|
-
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
274
|
-
|
|
275
|
-
SIMSIMD_PUBLIC void simsimd_sum_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
276
|
-
simsimd_f32_t *result) {
|
|
277
|
-
// The main loop:
|
|
278
|
-
simsimd_size_t i = 0;
|
|
279
|
-
for (; i + 8 <= n; i += 8) {
|
|
280
|
-
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
281
|
-
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
282
|
-
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
283
|
-
_mm256_storeu_ps(result + i, sum_vec);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
// The tail:
|
|
287
|
-
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
SIMSIMD_PUBLIC void simsimd_scale_f32_haswell(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
291
|
-
simsimd_f32_t *result) {
|
|
292
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
293
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
294
|
-
|
|
295
|
-
// The main loop:
|
|
296
|
-
simsimd_size_t i = 0;
|
|
297
|
-
for (; i + 8 <= n; i += 8) {
|
|
298
|
-
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
299
|
-
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
300
|
-
_mm256_storeu_ps(result + i, sum_vec);
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// The tail:
|
|
304
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i];
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
|
|
308
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
309
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
310
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
311
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
312
|
-
|
|
313
|
-
// There are are several special cases we may want to implement:
|
|
314
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
315
|
-
if (alpha == 1 && beta == 1) {
|
|
316
|
-
// In this case we can avoid expensive multiplications.
|
|
317
|
-
simsimd_sum_f32_haswell(a, b, n, result);
|
|
318
|
-
return;
|
|
319
|
-
}
|
|
320
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
321
|
-
else if (alpha == 0 || beta == 0) {
|
|
322
|
-
// In this case we can avoid half of the load instructions.
|
|
323
|
-
if (beta == 0) { simsimd_scale_f32_haswell(a, n, alpha, result); }
|
|
324
|
-
else { simsimd_scale_f32_haswell(b, n, beta, result); }
|
|
325
|
-
return;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
// The general case.
|
|
329
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
330
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
331
|
-
|
|
332
|
-
// The main loop:
|
|
333
|
-
simsimd_size_t i = 0;
|
|
334
|
-
for (; i + 8 <= n; i += 8) {
|
|
335
|
-
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
336
|
-
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
337
|
-
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
338
|
-
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
339
|
-
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
340
|
-
_mm256_storeu_ps(result + i, sum_vec);
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
// The tail:
|
|
344
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
SIMSIMD_PUBLIC void simsimd_sum_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
348
|
-
simsimd_f64_t *result) {
|
|
349
|
-
// The main loop:
|
|
350
|
-
simsimd_size_t i = 0;
|
|
351
|
-
for (; i + 4 <= n; i += 4) {
|
|
352
|
-
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
353
|
-
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
354
|
-
__m256d sum_vec = _mm256_add_pd(a_vec, b_vec);
|
|
355
|
-
_mm256_storeu_pd(result + i, sum_vec);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
// The tail:
|
|
359
|
-
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
SIMSIMD_PUBLIC void simsimd_scale_f64_haswell(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
363
|
-
simsimd_f64_t *result) {
|
|
364
|
-
__m256d alpha_vec = _mm256_set1_pd(alpha);
|
|
365
|
-
|
|
366
|
-
// The main loop:
|
|
367
|
-
simsimd_size_t i = 0;
|
|
368
|
-
for (; i + 4 <= n; i += 4) {
|
|
369
|
-
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
370
|
-
__m256d sum_vec = _mm256_mul_pd(a_vec, alpha_vec);
|
|
371
|
-
_mm256_storeu_pd(result + i, sum_vec);
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
// The tail:
|
|
375
|
-
for (; i < n; ++i) result[i] = alpha * a[i];
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
|
|
379
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
380
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
381
|
-
|
|
382
|
-
// There are are several special cases we may want to implement:
|
|
383
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
384
|
-
if (alpha == 1 && beta == 1) {
|
|
385
|
-
// In this case we can avoid expensive multiplications.
|
|
386
|
-
simsimd_sum_f64_haswell(a, b, n, result);
|
|
387
|
-
return;
|
|
388
|
-
}
|
|
389
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
390
|
-
else if (alpha == 0 || beta == 0) {
|
|
391
|
-
// In this case we can avoid half of the load instructions.
|
|
392
|
-
if (beta == 0) { simsimd_scale_f64_haswell(a, n, alpha, result); }
|
|
393
|
-
else { simsimd_scale_f64_haswell(b, n, beta, result); }
|
|
394
|
-
return;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// The general case.
|
|
398
|
-
__m256d alpha_vec = _mm256_set1_pd(alpha);
|
|
399
|
-
__m256d beta_vec = _mm256_set1_pd(beta);
|
|
400
|
-
|
|
401
|
-
// The main loop:
|
|
402
|
-
simsimd_size_t i = 0;
|
|
403
|
-
for (; i + 4 <= n; i += 4) {
|
|
404
|
-
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
405
|
-
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
406
|
-
__m256d a_scaled_vec = _mm256_mul_pd(a_vec, alpha_vec);
|
|
407
|
-
__m256d b_scaled_vec = _mm256_mul_pd(b_vec, beta_vec);
|
|
408
|
-
__m256d sum_vec = _mm256_add_pd(a_scaled_vec, b_scaled_vec);
|
|
409
|
-
_mm256_storeu_pd(result + i, sum_vec);
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
// The tail:
|
|
413
|
-
for (; i < n; ++i) result[i] = alpha * a[i] + beta * b[i];
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
SIMSIMD_PUBLIC void simsimd_sum_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
417
|
-
simsimd_f16_t *result) {
|
|
418
|
-
|
|
419
|
-
// The main loop:
|
|
420
|
-
simsimd_size_t i = 0;
|
|
421
|
-
for (; i + 8 <= n; i += 8) {
|
|
422
|
-
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
423
|
-
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
424
|
-
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
425
|
-
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
426
|
-
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
427
|
-
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
428
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
// The tail:
|
|
432
|
-
for (; i < n; ++i) {
|
|
433
|
-
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
434
|
-
simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
|
|
435
|
-
simsimd_f32_t sum = ai + bi;
|
|
436
|
-
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
SIMSIMD_PUBLIC void simsimd_scale_f16_haswell(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
441
|
-
simsimd_f16_t *result) {
|
|
442
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
443
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
444
|
-
|
|
445
|
-
// The main loop:
|
|
446
|
-
simsimd_size_t i = 0;
|
|
447
|
-
for (; i + 8 <= n; i += 8) {
|
|
448
|
-
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
449
|
-
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
450
|
-
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
451
|
-
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
452
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
// The tail:
|
|
456
|
-
for (; i < n; ++i) {
|
|
457
|
-
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
458
|
-
simsimd_f32_t sum = alpha_f32 * ai;
|
|
459
|
-
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
|
|
464
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
465
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
466
|
-
|
|
467
|
-
// There are are several special cases we may want to implement:
|
|
468
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
469
|
-
if (alpha == 1 && beta == 1) {
|
|
470
|
-
// In this case we can avoid expensive multiplications.
|
|
471
|
-
simsimd_sum_f16_haswell(a, b, n, result);
|
|
472
|
-
return;
|
|
473
|
-
}
|
|
474
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
475
|
-
else if (alpha == 0 || beta == 0) {
|
|
476
|
-
// In this case we can avoid half of the load instructions.
|
|
477
|
-
if (beta == 0) { simsimd_scale_f16_haswell(a, n, alpha, result); }
|
|
478
|
-
else { simsimd_scale_f16_haswell(b, n, beta, result); }
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
// The general case.
|
|
483
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
484
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
485
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
486
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
487
|
-
|
|
488
|
-
// The main loop:
|
|
489
|
-
simsimd_size_t i = 0;
|
|
490
|
-
for (; i + 8 <= n; i += 8) {
|
|
491
|
-
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
492
|
-
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
493
|
-
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
494
|
-
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
495
|
-
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
496
|
-
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
497
|
-
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
498
|
-
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
499
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
// The tail:
|
|
503
|
-
for (; i < n; ++i) {
|
|
504
|
-
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
505
|
-
simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
|
|
506
|
-
simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
|
|
507
|
-
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
SIMSIMD_PUBLIC void simsimd_sum_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
512
|
-
simsimd_bf16_t *result) {
|
|
513
|
-
// The main loop:
|
|
514
|
-
simsimd_size_t i = 0;
|
|
515
|
-
for (; i + 8 <= n; i += 8) {
|
|
516
|
-
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
517
|
-
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
518
|
-
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
519
|
-
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
520
|
-
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
521
|
-
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
522
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
// The tail:
|
|
526
|
-
for (; i < n; ++i) {
|
|
527
|
-
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
528
|
-
simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
|
|
529
|
-
simsimd_f32_t sum = ai + bi;
|
|
530
|
-
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
SIMSIMD_PUBLIC void simsimd_scale_bf16_haswell(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
535
|
-
simsimd_bf16_t *result) {
|
|
536
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
537
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
538
|
-
|
|
539
|
-
// The main loop:
|
|
540
|
-
simsimd_size_t i = 0;
|
|
541
|
-
for (; i + 8 <= n; i += 8) {
|
|
542
|
-
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
543
|
-
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
544
|
-
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
545
|
-
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
546
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
// The tail:
|
|
550
|
-
for (; i < n; ++i) {
|
|
551
|
-
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
552
|
-
simsimd_f32_t sum = alpha_f32 * ai;
|
|
553
|
-
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
|
|
558
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
559
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
560
|
-
|
|
561
|
-
// There are are several special cases we may want to implement:
|
|
562
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
563
|
-
if (alpha == 1 && beta == 1) {
|
|
564
|
-
// In this case we can avoid expensive multiplications.
|
|
565
|
-
simsimd_sum_bf16_haswell(a, b, n, result);
|
|
566
|
-
return;
|
|
567
|
-
}
|
|
568
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
569
|
-
else if (alpha == 0 || beta == 0) {
|
|
570
|
-
// In this case we can avoid half of the load instructions.
|
|
571
|
-
if (beta == 0) { simsimd_scale_bf16_haswell(a, n, alpha, result); }
|
|
572
|
-
else { simsimd_scale_bf16_haswell(b, n, beta, result); }
|
|
573
|
-
return;
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
// The general case.
|
|
577
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
578
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
579
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
580
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
581
|
-
|
|
582
|
-
// The main loop:
|
|
583
|
-
simsimd_size_t i = 0;
|
|
584
|
-
for (; i + 8 <= n; i += 8) {
|
|
585
|
-
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
586
|
-
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
587
|
-
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
588
|
-
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
589
|
-
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
590
|
-
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
591
|
-
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
592
|
-
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
593
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
// The tail:
|
|
597
|
-
for (; i < n; ++i) {
|
|
598
|
-
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
599
|
-
simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
|
|
600
|
-
simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
|
|
601
|
-
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
|
|
606
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
|
|
607
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
608
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
609
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
610
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
611
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
612
|
-
|
|
613
|
-
// The main loop:
|
|
614
|
-
simsimd_size_t i = 0;
|
|
615
|
-
for (; i + 8 <= n; i += 8) {
|
|
616
|
-
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
617
|
-
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
618
|
-
__m256 c_vec = _mm256_loadu_ps(c + i);
|
|
619
|
-
__m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
|
|
620
|
-
__m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
|
|
621
|
-
__m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
|
|
622
|
-
__m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
|
|
623
|
-
_mm256_storeu_ps(result + i, sum_vec);
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
// The tail:
|
|
627
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
|
|
631
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, //
|
|
632
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
633
|
-
__m256d alpha_vec = _mm256_set1_pd(alpha);
|
|
634
|
-
__m256d beta_vec = _mm256_set1_pd(beta);
|
|
635
|
-
|
|
636
|
-
// The main loop:
|
|
637
|
-
simsimd_size_t i = 0;
|
|
638
|
-
for (; i + 4 <= n; i += 4) {
|
|
639
|
-
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
640
|
-
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
641
|
-
__m256d c_vec = _mm256_loadu_pd(c + i);
|
|
642
|
-
__m256d ab_vec = _mm256_mul_pd(a_vec, b_vec);
|
|
643
|
-
__m256d ab_scaled_vec = _mm256_mul_pd(ab_vec, alpha_vec);
|
|
644
|
-
__m256d c_scaled_vec = _mm256_mul_pd(c_vec, beta_vec);
|
|
645
|
-
__m256d sum_vec = _mm256_add_pd(ab_scaled_vec, c_scaled_vec);
|
|
646
|
-
_mm256_storeu_pd(result + i, sum_vec);
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
// The tail:
|
|
650
|
-
for (; i < n; ++i) result[i] = alpha * a[i] * b[i] + beta * c[i];
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
|
|
654
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
|
|
655
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
656
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
657
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
658
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
659
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
660
|
-
|
|
661
|
-
// The main loop:
|
|
662
|
-
simsimd_size_t i = 0;
|
|
663
|
-
for (; i + 8 <= n; i += 8) {
|
|
664
|
-
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
665
|
-
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
666
|
-
__m128i c_f16 = _mm_lddqu_si128((__m128i const *)(c + i));
|
|
667
|
-
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
668
|
-
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
669
|
-
__m256 c_vec = _mm256_cvtph_ps(c_f16);
|
|
670
|
-
__m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
|
|
671
|
-
__m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
|
|
672
|
-
__m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
|
|
673
|
-
__m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
|
|
674
|
-
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
675
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
// The tail:
|
|
679
|
-
for (; i < n; ++i) {
|
|
680
|
-
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
681
|
-
simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
|
|
682
|
-
simsimd_f32_t ci = SIMSIMD_F16_TO_F32(c + i);
|
|
683
|
-
simsimd_f32_t sum = alpha * ai * bi + beta * ci;
|
|
684
|
-
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
|
|
689
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
|
|
690
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
691
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
692
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
693
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
694
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
695
|
-
|
|
696
|
-
// The main loop:
|
|
697
|
-
simsimd_size_t i = 0;
|
|
698
|
-
for (; i + 8 <= n; i += 8) {
|
|
699
|
-
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
700
|
-
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
701
|
-
__m128i c_bf16 = _mm_lddqu_si128((__m128i const *)(c + i));
|
|
702
|
-
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
703
|
-
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
704
|
-
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(c_bf16);
|
|
705
|
-
__m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
|
|
706
|
-
__m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
|
|
707
|
-
__m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
|
|
708
|
-
__m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
|
|
709
|
-
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
710
|
-
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
// The tail:
|
|
714
|
-
for (; i < n; ++i) {
|
|
715
|
-
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
716
|
-
simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
|
|
717
|
-
simsimd_f32_t ci = SIMSIMD_BF16_TO_F32(c + i);
|
|
718
|
-
simsimd_f32_t sum = alpha * ai * bi + beta * ci;
|
|
719
|
-
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
SIMSIMD_PUBLIC void simsimd_sum_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
724
|
-
simsimd_i8_t *result) {
|
|
725
|
-
// The main loop:
|
|
726
|
-
simsimd_size_t i = 0;
|
|
727
|
-
for (; i + 32 <= n; i += 32) {
|
|
728
|
-
__m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
|
|
729
|
-
__m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
|
|
730
|
-
__m256i sum_vec = _mm256_adds_epi8(a_vec, b_vec);
|
|
731
|
-
_mm256_storeu_si256((__m256i *)(result + i), sum_vec);
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
// The tail:
|
|
735
|
-
for (; i < n; ++i) {
|
|
736
|
-
simsimd_f32_t ai = a[i], bi = b[i];
|
|
737
|
-
simsimd_f32_t sum = ai + bi;
|
|
738
|
-
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
SIMSIMD_PUBLIC void simsimd_scale_i8_haswell(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
743
|
-
simsimd_i8_t *result) {
|
|
744
|
-
|
|
745
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
746
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
747
|
-
int sum_i32s[8], a_i32s[8];
|
|
748
|
-
|
|
749
|
-
// The main loop:
|
|
750
|
-
simsimd_size_t i = 0;
|
|
751
|
-
for (; i + 8 <= n; i += 8) {
|
|
752
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
753
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
754
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
755
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
756
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
757
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
758
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
759
|
-
// The normal part.
|
|
760
|
-
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
761
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
762
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
763
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
|
|
764
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
|
|
765
|
-
// Export into a serial buffer.
|
|
766
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
767
|
-
result[i + 0] = (simsimd_i8_t)sum_i32s[0];
|
|
768
|
-
result[i + 1] = (simsimd_i8_t)sum_i32s[1];
|
|
769
|
-
result[i + 2] = (simsimd_i8_t)sum_i32s[2];
|
|
770
|
-
result[i + 3] = (simsimd_i8_t)sum_i32s[3];
|
|
771
|
-
result[i + 4] = (simsimd_i8_t)sum_i32s[4];
|
|
772
|
-
result[i + 5] = (simsimd_i8_t)sum_i32s[5];
|
|
773
|
-
result[i + 6] = (simsimd_i8_t)sum_i32s[6];
|
|
774
|
-
result[i + 7] = (simsimd_i8_t)sum_i32s[7];
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
// The tail:
|
|
778
|
-
for (; i < n; ++i) {
|
|
779
|
-
simsimd_f32_t ai = a[i];
|
|
780
|
-
simsimd_f32_t sum = alpha_f32 * ai;
|
|
781
|
-
SIMSIMD_F32_TO_I8(sum, result + i);
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
|
|
786
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
787
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
788
|
-
|
|
789
|
-
// There are are several special cases we may want to implement:
|
|
790
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
791
|
-
if (alpha == 1 && beta == 1) {
|
|
792
|
-
// In this case we can avoid expensive multiplications.
|
|
793
|
-
simsimd_sum_i8_haswell(a, b, n, result);
|
|
794
|
-
return;
|
|
795
|
-
}
|
|
796
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
797
|
-
else if (alpha == 0 || beta == 0) {
|
|
798
|
-
// In this case we can avoid half of the load instructions.
|
|
799
|
-
if (beta == 0) { simsimd_scale_i8_haswell(a, n, alpha, result); }
|
|
800
|
-
else { simsimd_scale_i8_haswell(b, n, beta, result); }
|
|
801
|
-
return;
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
// The general case.
|
|
805
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
806
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
807
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
808
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
809
|
-
int sum_i32s[8], a_i32s[8], b_i32s[8];
|
|
810
|
-
|
|
811
|
-
// The main loop:
|
|
812
|
-
simsimd_size_t i = 0;
|
|
813
|
-
for (; i + 8 <= n; i += 8) {
|
|
814
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
815
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
816
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
817
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
818
|
-
b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
|
|
819
|
-
b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
|
|
820
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
821
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
822
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
823
|
-
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
824
|
-
// The normal part.
|
|
825
|
-
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
826
|
-
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
827
|
-
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
828
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
829
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
830
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
|
|
831
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
|
|
832
|
-
// Export into a serial buffer.
|
|
833
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
834
|
-
result[i + 0] = (simsimd_i8_t)sum_i32s[0];
|
|
835
|
-
result[i + 1] = (simsimd_i8_t)sum_i32s[1];
|
|
836
|
-
result[i + 2] = (simsimd_i8_t)sum_i32s[2];
|
|
837
|
-
result[i + 3] = (simsimd_i8_t)sum_i32s[3];
|
|
838
|
-
result[i + 4] = (simsimd_i8_t)sum_i32s[4];
|
|
839
|
-
result[i + 5] = (simsimd_i8_t)sum_i32s[5];
|
|
840
|
-
result[i + 6] = (simsimd_i8_t)sum_i32s[6];
|
|
841
|
-
result[i + 7] = (simsimd_i8_t)sum_i32s[7];
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
// The tail:
|
|
845
|
-
for (; i < n; ++i) {
|
|
846
|
-
simsimd_f32_t ai = a[i], bi = b[i];
|
|
847
|
-
simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
|
|
848
|
-
SIMSIMD_F32_TO_I8(sum, result + i);
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
|
-
|
|
852
|
-
SIMSIMD_PUBLIC void simsimd_sum_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
853
|
-
simsimd_u8_t *result) {
|
|
854
|
-
// The main loop:
|
|
855
|
-
simsimd_size_t i = 0;
|
|
856
|
-
for (; i + 32 <= n; i += 32) {
|
|
857
|
-
__m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
|
|
858
|
-
__m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
|
|
859
|
-
__m256i sum_vec = _mm256_adds_epu8(a_vec, b_vec);
|
|
860
|
-
_mm256_storeu_si256((__m256i *)(result + i), sum_vec);
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
// The tail:
|
|
864
|
-
for (; i < n; ++i) {
|
|
865
|
-
simsimd_f32_t ai = a[i], bi = b[i];
|
|
866
|
-
simsimd_f32_t sum = ai + bi;
|
|
867
|
-
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
SIMSIMD_PUBLIC void simsimd_scale_u8_haswell(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
872
|
-
simsimd_u8_t *result) {
|
|
873
|
-
|
|
874
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
875
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
876
|
-
int sum_i32s[8], a_i32s[8];
|
|
877
|
-
|
|
878
|
-
// The main loop:
|
|
879
|
-
simsimd_size_t i = 0;
|
|
880
|
-
for (; i + 8 <= n; i += 8) {
|
|
881
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
882
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
883
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
884
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
885
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
886
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
887
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
888
|
-
// The normal part.
|
|
889
|
-
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
890
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
891
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
892
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
|
|
893
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
|
|
894
|
-
// Export into a serial buffer.
|
|
895
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
896
|
-
result[i + 0] = (simsimd_u8_t)sum_i32s[0];
|
|
897
|
-
result[i + 1] = (simsimd_u8_t)sum_i32s[1];
|
|
898
|
-
result[i + 2] = (simsimd_u8_t)sum_i32s[2];
|
|
899
|
-
result[i + 3] = (simsimd_u8_t)sum_i32s[3];
|
|
900
|
-
result[i + 4] = (simsimd_u8_t)sum_i32s[4];
|
|
901
|
-
result[i + 5] = (simsimd_u8_t)sum_i32s[5];
|
|
902
|
-
result[i + 6] = (simsimd_u8_t)sum_i32s[6];
|
|
903
|
-
result[i + 7] = (simsimd_u8_t)sum_i32s[7];
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
// The tail:
|
|
907
|
-
for (; i < n; ++i) {
|
|
908
|
-
simsimd_f32_t ai = a[i];
|
|
909
|
-
simsimd_f32_t sum = alpha_f32 * ai;
|
|
910
|
-
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
|
|
915
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
916
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
917
|
-
|
|
918
|
-
// There are are several special cases we may want to implement:
|
|
919
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
920
|
-
if (alpha == 1 && beta == 1) {
|
|
921
|
-
// In this case we can avoid expensive multiplications.
|
|
922
|
-
simsimd_sum_u8_haswell(a, b, n, result);
|
|
923
|
-
return;
|
|
924
|
-
}
|
|
925
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
926
|
-
else if (alpha == 0 || beta == 0) {
|
|
927
|
-
// In this case we can avoid half of the load instructions.
|
|
928
|
-
if (beta == 0) { simsimd_scale_u8_haswell(a, n, alpha, result); }
|
|
929
|
-
else { simsimd_scale_u8_haswell(b, n, beta, result); }
|
|
930
|
-
return;
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
// The general case.
|
|
934
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
935
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
936
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
937
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
938
|
-
int sum_i32s[8], a_i32s[8], b_i32s[8];
|
|
939
|
-
|
|
940
|
-
// The main loop:
|
|
941
|
-
simsimd_size_t i = 0;
|
|
942
|
-
for (; i + 8 <= n; i += 8) {
|
|
943
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
944
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
945
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
946
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
947
|
-
b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
|
|
948
|
-
b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
|
|
949
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
950
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
951
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
952
|
-
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
953
|
-
// The normal part.
|
|
954
|
-
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
955
|
-
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
956
|
-
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
957
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
958
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
959
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
|
|
960
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
|
|
961
|
-
// Export into a serial buffer.
|
|
962
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
963
|
-
result[i + 0] = (simsimd_u8_t)sum_i32s[0];
|
|
964
|
-
result[i + 1] = (simsimd_u8_t)sum_i32s[1];
|
|
965
|
-
result[i + 2] = (simsimd_u8_t)sum_i32s[2];
|
|
966
|
-
result[i + 3] = (simsimd_u8_t)sum_i32s[3];
|
|
967
|
-
result[i + 4] = (simsimd_u8_t)sum_i32s[4];
|
|
968
|
-
result[i + 5] = (simsimd_u8_t)sum_i32s[5];
|
|
969
|
-
result[i + 6] = (simsimd_u8_t)sum_i32s[6];
|
|
970
|
-
result[i + 7] = (simsimd_u8_t)sum_i32s[7];
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
// The tail:
|
|
974
|
-
for (; i < n; ++i) {
|
|
975
|
-
simsimd_f32_t ai = a[i], bi = b[i];
|
|
976
|
-
simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
|
|
977
|
-
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
|
|
981
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
|
|
982
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
|
|
983
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
984
|
-
|
|
985
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
986
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
987
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
988
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
989
|
-
int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
|
|
990
|
-
|
|
991
|
-
// The main loop:
|
|
992
|
-
simsimd_size_t i = 0;
|
|
993
|
-
for (; i + 8 <= n; i += 8) {
|
|
994
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
995
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
996
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
997
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
998
|
-
b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
|
|
999
|
-
b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
|
|
1000
|
-
c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
|
|
1001
|
-
c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
|
|
1002
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
1003
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
1004
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
1005
|
-
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
1006
|
-
__m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
|
|
1007
|
-
// The normal part.
|
|
1008
|
-
__m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
|
|
1009
|
-
__m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
|
|
1010
|
-
__m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
|
|
1011
|
-
__m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
|
|
1012
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
1013
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
1014
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
|
|
1015
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
|
|
1016
|
-
// Export into a serial buffer.
|
|
1017
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
1018
|
-
result[i + 0] = (simsimd_i8_t)sum_i32s[0];
|
|
1019
|
-
result[i + 1] = (simsimd_i8_t)sum_i32s[1];
|
|
1020
|
-
result[i + 2] = (simsimd_i8_t)sum_i32s[2];
|
|
1021
|
-
result[i + 3] = (simsimd_i8_t)sum_i32s[3];
|
|
1022
|
-
result[i + 4] = (simsimd_i8_t)sum_i32s[4];
|
|
1023
|
-
result[i + 5] = (simsimd_i8_t)sum_i32s[5];
|
|
1024
|
-
result[i + 6] = (simsimd_i8_t)sum_i32s[6];
|
|
1025
|
-
result[i + 7] = (simsimd_i8_t)sum_i32s[7];
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
// The tail:
|
|
1029
|
-
for (; i < n; ++i) {
|
|
1030
|
-
simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
|
|
1031
|
-
simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
|
|
1032
|
-
SIMSIMD_F32_TO_I8(sum, result + i);
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
|
|
1037
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
|
|
1038
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
1039
|
-
|
|
1040
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
1041
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
1042
|
-
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
1043
|
-
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
1044
|
-
int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
|
|
1045
|
-
|
|
1046
|
-
// The main loop:
|
|
1047
|
-
simsimd_size_t i = 0;
|
|
1048
|
-
for (; i + 8 <= n; i += 8) {
|
|
1049
|
-
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
1050
|
-
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
1051
|
-
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
1052
|
-
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
1053
|
-
b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
|
|
1054
|
-
b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
|
|
1055
|
-
c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
|
|
1056
|
-
c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
|
|
1057
|
-
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
1058
|
-
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
1059
|
-
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
1060
|
-
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
1061
|
-
__m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
|
|
1062
|
-
// The normal part.
|
|
1063
|
-
__m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
|
|
1064
|
-
__m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
|
|
1065
|
-
__m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
|
|
1066
|
-
__m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
|
|
1067
|
-
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
1068
|
-
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
1069
|
-
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
|
|
1070
|
-
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
|
|
1071
|
-
// Export into a serial buffer.
|
|
1072
|
-
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
1073
|
-
result[i + 0] = (simsimd_u8_t)sum_i32s[0];
|
|
1074
|
-
result[i + 1] = (simsimd_u8_t)sum_i32s[1];
|
|
1075
|
-
result[i + 2] = (simsimd_u8_t)sum_i32s[2];
|
|
1076
|
-
result[i + 3] = (simsimd_u8_t)sum_i32s[3];
|
|
1077
|
-
result[i + 4] = (simsimd_u8_t)sum_i32s[4];
|
|
1078
|
-
result[i + 5] = (simsimd_u8_t)sum_i32s[5];
|
|
1079
|
-
result[i + 6] = (simsimd_u8_t)sum_i32s[6];
|
|
1080
|
-
result[i + 7] = (simsimd_u8_t)sum_i32s[7];
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
// The tail:
|
|
1084
|
-
for (; i < n; ++i) {
|
|
1085
|
-
simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
|
|
1086
|
-
simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
|
|
1087
|
-
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
1088
|
-
}
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
|
-
#pragma clang attribute pop
|
|
1092
|
-
#pragma GCC pop_options
|
|
1093
|
-
#endif // SIMSIMD_TARGET_HASWELL
|
|
1094
|
-
|
|
1095
|
-
#if SIMSIMD_TARGET_SKYLAKE
|
|
1096
|
-
#pragma GCC push_options
|
|
1097
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
1098
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
1099
|
-
|
|
1100
|
-
SIMSIMD_PUBLIC void simsimd_sum_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1101
|
-
simsimd_f64_t *result) {
|
|
1102
|
-
__m512d a_vec, b_vec, sum_vec;
|
|
1103
|
-
__mmask8 mask = 0xFF;
|
|
1104
|
-
simsimd_sum_f64_skylake_cycle:
|
|
1105
|
-
if (n < 8) {
|
|
1106
|
-
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1107
|
-
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1108
|
-
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1109
|
-
n = 0;
|
|
1110
|
-
}
|
|
1111
|
-
else {
|
|
1112
|
-
a_vec = _mm512_loadu_pd(a);
|
|
1113
|
-
b_vec = _mm512_loadu_pd(b);
|
|
1114
|
-
a += 8, b += 8, n -= 8;
|
|
1115
|
-
}
|
|
1116
|
-
sum_vec = _mm512_add_pd(a_vec, b_vec);
|
|
1117
|
-
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1118
|
-
result += 8;
|
|
1119
|
-
if (n) goto simsimd_sum_f64_skylake_cycle;
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
SIMSIMD_PUBLIC void simsimd_scale_f64_skylake(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1123
|
-
simsimd_f64_t *result) {
|
|
1124
|
-
__m512d alpha_vec = _mm512_set1_pd(alpha);
|
|
1125
|
-
__m512d a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
1126
|
-
__mmask8 mask = 0xFF;
|
|
1127
|
-
simsimd_scale_f64_skylake_cycle:
|
|
1128
|
-
if (n < 8) {
|
|
1129
|
-
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1130
|
-
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1131
|
-
n = 0;
|
|
1132
|
-
}
|
|
1133
|
-
else {
|
|
1134
|
-
a_vec = _mm512_loadu_pd(a);
|
|
1135
|
-
a += 8, n -= 8;
|
|
1136
|
-
}
|
|
1137
|
-
sum_vec = _mm512_mul_pd(a_vec, alpha_vec);
|
|
1138
|
-
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1139
|
-
result += 8;
|
|
1140
|
-
if (n) goto simsimd_scale_f64_skylake_cycle;
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
|
|
1144
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
1145
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
1146
|
-
|
|
1147
|
-
// There are are several special cases we may want to implement:
|
|
1148
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1149
|
-
if (alpha == 1 && beta == 1) {
|
|
1150
|
-
// In this case we can avoid expensive multiplications.
|
|
1151
|
-
simsimd_sum_f64_skylake(a, b, n, result);
|
|
1152
|
-
return;
|
|
1153
|
-
}
|
|
1154
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1155
|
-
else if (alpha == 0 || beta == 0) {
|
|
1156
|
-
// In this case we can avoid half of the load instructions.
|
|
1157
|
-
if (beta == 0) { simsimd_scale_f64_skylake(a, n, alpha, result); }
|
|
1158
|
-
else { simsimd_scale_f64_skylake(b, n, beta, result); }
|
|
1159
|
-
return;
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
// The general case.
|
|
1163
|
-
__m512d alpha_vec = _mm512_set1_pd(alpha);
|
|
1164
|
-
__m512d beta_vec = _mm512_set1_pd(beta);
|
|
1165
|
-
__m512d a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
1166
|
-
__mmask8 mask = 0xFF;
|
|
1167
|
-
simsimd_wsum_f64_skylake_cycle:
|
|
1168
|
-
if (n < 8) {
|
|
1169
|
-
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1170
|
-
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1171
|
-
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1172
|
-
n = 0;
|
|
1173
|
-
}
|
|
1174
|
-
else {
|
|
1175
|
-
a_vec = _mm512_loadu_pd(a);
|
|
1176
|
-
b_vec = _mm512_loadu_pd(b);
|
|
1177
|
-
a += 8, b += 8, n -= 8;
|
|
1178
|
-
}
|
|
1179
|
-
a_scaled_vec = _mm512_mul_pd(a_vec, alpha_vec);
|
|
1180
|
-
sum_vec = _mm512_fmadd_pd(b_vec, beta_vec, a_scaled_vec);
|
|
1181
|
-
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1182
|
-
result += 8;
|
|
1183
|
-
if (n) goto simsimd_wsum_f64_skylake_cycle;
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
SIMSIMD_PUBLIC void simsimd_sum_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1187
|
-
simsimd_f32_t *result) {
|
|
1188
|
-
__m512 a_vec, b_vec, sum_vec;
|
|
1189
|
-
__mmask16 mask = 0xFFFF;
|
|
1190
|
-
|
|
1191
|
-
simsimd_sum_f32_skylake_cycle:
|
|
1192
|
-
if (n < 16) {
|
|
1193
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1194
|
-
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1195
|
-
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1196
|
-
n = 0;
|
|
1197
|
-
}
|
|
1198
|
-
else {
|
|
1199
|
-
a_vec = _mm512_loadu_ps(a);
|
|
1200
|
-
b_vec = _mm512_loadu_ps(b);
|
|
1201
|
-
a += 16, b += 16, n -= 16;
|
|
1202
|
-
}
|
|
1203
|
-
sum_vec = _mm512_add_ps(a_vec, b_vec);
|
|
1204
|
-
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1205
|
-
result += 16;
|
|
1206
|
-
if (n) goto simsimd_sum_f32_skylake_cycle;
|
|
1207
|
-
}
|
|
1208
|
-
|
|
1209
|
-
SIMSIMD_PUBLIC void simsimd_scale_f32_skylake(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1210
|
-
simsimd_f32_t *result) {
|
|
1211
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1212
|
-
__m512 a_vec, sum_vec;
|
|
1213
|
-
__mmask16 mask = 0xFFFF;
|
|
1214
|
-
|
|
1215
|
-
simsimd_scale_f32_skylake_cycle:
|
|
1216
|
-
if (n < 16) {
|
|
1217
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1218
|
-
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1219
|
-
n = 0;
|
|
1220
|
-
}
|
|
1221
|
-
else {
|
|
1222
|
-
a_vec = _mm512_loadu_ps(a);
|
|
1223
|
-
a += 16, n -= 16;
|
|
1224
|
-
}
|
|
1225
|
-
sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1226
|
-
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1227
|
-
result += 16;
|
|
1228
|
-
if (n) goto simsimd_scale_f32_skylake_cycle;
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
|
|
1232
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
1233
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
1234
|
-
|
|
1235
|
-
// There are are several special cases we may want to implement:
|
|
1236
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1237
|
-
if (alpha == 1 && beta == 1) {
|
|
1238
|
-
// In this case we can avoid expensive multiplications.
|
|
1239
|
-
simsimd_sum_f32_skylake(a, b, n, result);
|
|
1240
|
-
return;
|
|
1241
|
-
}
|
|
1242
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1243
|
-
else if (alpha == 0 || beta == 0) {
|
|
1244
|
-
// In this case we can avoid half of the load instructions.
|
|
1245
|
-
if (beta == 0) { simsimd_scale_f32_skylake(a, n, alpha, result); }
|
|
1246
|
-
else { simsimd_scale_f32_skylake(b, n, beta, result); }
|
|
1247
|
-
return;
|
|
1248
|
-
}
|
|
1249
|
-
|
|
1250
|
-
// The general case.
|
|
1251
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1252
|
-
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
1253
|
-
__m512 a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
1254
|
-
__mmask16 mask = 0xFFFF;
|
|
1255
|
-
simsimd_wsum_f32_skylake_cycle:
|
|
1256
|
-
if (n < 16) {
|
|
1257
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1258
|
-
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1259
|
-
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1260
|
-
n = 0;
|
|
1261
|
-
}
|
|
1262
|
-
else {
|
|
1263
|
-
a_vec = _mm512_loadu_ps(a);
|
|
1264
|
-
b_vec = _mm512_loadu_ps(b);
|
|
1265
|
-
a += 16, b += 16, n -= 16;
|
|
1266
|
-
}
|
|
1267
|
-
a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1268
|
-
sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
|
|
1269
|
-
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1270
|
-
result += 16;
|
|
1271
|
-
if (n) goto simsimd_wsum_f32_skylake_cycle;
|
|
1272
|
-
}
|
|
1273
|
-
|
|
1274
|
-
SIMSIMD_PUBLIC void simsimd_sum_bf16_skylake(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1275
|
-
simsimd_bf16_t *result) {
|
|
1276
|
-
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
1277
|
-
__m512 a_vec, b_vec, sum_vec;
|
|
1278
|
-
__mmask16 mask = 0xFFFF;
|
|
1279
|
-
simsimd_sum_bf16_skylake_cycle:
|
|
1280
|
-
if (n < 16) {
|
|
1281
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1282
|
-
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1283
|
-
b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
|
|
1284
|
-
n = 0;
|
|
1285
|
-
}
|
|
1286
|
-
else {
|
|
1287
|
-
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1288
|
-
b_bf16_vec = _mm256_loadu_epi16(b);
|
|
1289
|
-
a += 16, b += 16, n -= 16;
|
|
1290
|
-
}
|
|
1291
|
-
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1292
|
-
b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
|
|
1293
|
-
sum_vec = _mm512_add_ps(a_vec, b_vec);
|
|
1294
|
-
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1295
|
-
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1296
|
-
result += 16;
|
|
1297
|
-
if (n) goto simsimd_sum_bf16_skylake_cycle;
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
SIMSIMD_PUBLIC void simsimd_scale_bf16_skylake(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1301
|
-
simsimd_bf16_t *result) {
|
|
1302
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1303
|
-
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
1304
|
-
__m512 a_vec, b_vec, sum_vec;
|
|
1305
|
-
__mmask16 mask = 0xFFFF;
|
|
1306
|
-
simsimd_wsum_bf16_skylake_cycle:
|
|
1307
|
-
if (n < 16) {
|
|
1308
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1309
|
-
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1310
|
-
n = 0;
|
|
1311
|
-
}
|
|
1312
|
-
else {
|
|
1313
|
-
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1314
|
-
a += 16, n -= 16;
|
|
1315
|
-
}
|
|
1316
|
-
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1317
|
-
sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1318
|
-
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1319
|
-
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1320
|
-
result += 16;
|
|
1321
|
-
if (n) goto simsimd_wsum_bf16_skylake_cycle;
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
|
|
1325
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
1326
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
1327
|
-
|
|
1328
|
-
// There are are several special cases we may want to implement:
|
|
1329
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1330
|
-
if (alpha == 1 && beta == 1) {
|
|
1331
|
-
// In this case we can avoid expensive multiplications.
|
|
1332
|
-
simsimd_sum_bf16_skylake(a, b, n, result);
|
|
1333
|
-
return;
|
|
1334
|
-
}
|
|
1335
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1336
|
-
else if (alpha == 0 || beta == 0) {
|
|
1337
|
-
// In this case we can avoid half of the load instructions.
|
|
1338
|
-
if (beta == 0) { simsimd_scale_bf16_skylake(a, n, alpha, result); }
|
|
1339
|
-
else { simsimd_scale_bf16_skylake(b, n, beta, result); }
|
|
1340
|
-
return;
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
// The general case.
|
|
1344
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1345
|
-
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
1346
|
-
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
1347
|
-
__m512 a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
1348
|
-
__mmask16 mask = 0xFFFF;
|
|
1349
|
-
simsimd_wsum_bf16_skylake_cycle:
|
|
1350
|
-
if (n < 16) {
|
|
1351
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1352
|
-
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1353
|
-
b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
|
|
1354
|
-
n = 0;
|
|
1355
|
-
}
|
|
1356
|
-
else {
|
|
1357
|
-
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1358
|
-
b_bf16_vec = _mm256_loadu_epi16(b);
|
|
1359
|
-
a += 16, b += 16, n -= 16;
|
|
1360
|
-
}
|
|
1361
|
-
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1362
|
-
b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
|
|
1363
|
-
a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1364
|
-
sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
|
|
1365
|
-
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1366
|
-
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1367
|
-
result += 16;
|
|
1368
|
-
if (n) goto simsimd_wsum_bf16_skylake_cycle;
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
|
|
1372
|
-
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
|
|
1373
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
1374
|
-
__m512d alpha_vec = _mm512_set1_pd(alpha);
|
|
1375
|
-
__m512d beta_vec = _mm512_set1_pd(beta);
|
|
1376
|
-
__m512d a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
|
|
1377
|
-
__mmask8 mask = 0xFF;
|
|
1378
|
-
|
|
1379
|
-
simsimd_fma_f64_skylake_cycle:
|
|
1380
|
-
if (n < 8) {
|
|
1381
|
-
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1382
|
-
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1383
|
-
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1384
|
-
c_vec = _mm512_maskz_loadu_pd(mask, c);
|
|
1385
|
-
n = 0;
|
|
1386
|
-
}
|
|
1387
|
-
else {
|
|
1388
|
-
a_vec = _mm512_loadu_pd(a);
|
|
1389
|
-
b_vec = _mm512_loadu_pd(b);
|
|
1390
|
-
c_vec = _mm512_loadu_pd(c);
|
|
1391
|
-
a += 8, b += 8, c += 8, n -= 8;
|
|
1392
|
-
}
|
|
1393
|
-
ab_vec = _mm512_mul_pd(a_vec, b_vec);
|
|
1394
|
-
ab_scaled_vec = _mm512_mul_pd(ab_vec, alpha_vec);
|
|
1395
|
-
sum_vec = _mm512_fmadd_pd(c_vec, beta_vec, ab_scaled_vec);
|
|
1396
|
-
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1397
|
-
result += 8;
|
|
1398
|
-
if (n) goto simsimd_fma_f64_skylake_cycle;
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
|
|
1402
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
|
|
1403
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
1404
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1405
|
-
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
1406
|
-
__m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
|
|
1407
|
-
__mmask16 mask = 0xFFFF;
|
|
1408
|
-
|
|
1409
|
-
simsimd_fma_f32_skylake_cycle:
|
|
1410
|
-
if (n < 16) {
|
|
1411
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1412
|
-
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1413
|
-
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1414
|
-
c_vec = _mm512_maskz_loadu_ps(mask, c);
|
|
1415
|
-
n = 0;
|
|
1416
|
-
}
|
|
1417
|
-
else {
|
|
1418
|
-
a_vec = _mm512_loadu_ps(a);
|
|
1419
|
-
b_vec = _mm512_loadu_ps(b);
|
|
1420
|
-
c_vec = _mm512_loadu_ps(c);
|
|
1421
|
-
a += 16, b += 16, c += 16, n -= 16;
|
|
1422
|
-
}
|
|
1423
|
-
ab_vec = _mm512_mul_ps(a_vec, b_vec);
|
|
1424
|
-
ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
|
|
1425
|
-
sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
|
|
1426
|
-
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1427
|
-
result += 16;
|
|
1428
|
-
if (n) goto simsimd_fma_f32_skylake_cycle;
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
|
|
1432
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
|
|
1433
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
1434
|
-
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1435
|
-
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
1436
|
-
__m256i a_bf16_vec, b_bf16_vec, c_bf16_vec, sum_bf16_vec;
|
|
1437
|
-
__m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
|
|
1438
|
-
__mmask16 mask = 0xFFFF;
|
|
1439
|
-
|
|
1440
|
-
simsimd_fma_bf16_skylake_cycle:
|
|
1441
|
-
if (n < 16) {
|
|
1442
|
-
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1443
|
-
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1444
|
-
b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
|
|
1445
|
-
c_bf16_vec = _mm256_maskz_loadu_epi16(mask, c);
|
|
1446
|
-
n = 0;
|
|
1447
|
-
}
|
|
1448
|
-
else {
|
|
1449
|
-
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1450
|
-
b_bf16_vec = _mm256_loadu_epi16(b);
|
|
1451
|
-
c_bf16_vec = _mm256_loadu_epi16(c);
|
|
1452
|
-
a += 16, b += 16, c += 16, n -= 16;
|
|
1453
|
-
}
|
|
1454
|
-
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1455
|
-
b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
|
|
1456
|
-
c_vec = _simsimd_bf16x16_to_f32x16_skylake(c_bf16_vec);
|
|
1457
|
-
ab_vec = _mm512_mul_ps(a_vec, b_vec);
|
|
1458
|
-
ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
|
|
1459
|
-
sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
|
|
1460
|
-
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1461
|
-
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1462
|
-
result += 16;
|
|
1463
|
-
if (n) goto simsimd_fma_bf16_skylake_cycle;
|
|
1464
|
-
}
|
|
1465
|
-
|
|
1466
|
-
#pragma clang attribute pop
|
|
1467
|
-
#pragma GCC pop_options
|
|
1468
|
-
#endif // SIMSIMD_TARGET_SKYLAKE
|
|
1469
|
-
|
|
1470
|
-
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1471
|
-
#pragma GCC push_options
|
|
1472
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16", "f16c")
|
|
1473
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16,f16c"))), \
|
|
1474
|
-
apply_to = function)
|
|
1475
|
-
|
|
1476
|
-
/**
|
|
1477
|
-
* Using `_mm512_set1_ph((_Float16)1.f)` results in compilation warnings if we are pedantic.
|
|
1478
|
-
* https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/details-about-intrinsics-for-half-floats.html
|
|
1479
|
-
*/
|
|
1480
|
-
SIMSIMD_INTERNAL __m512h _mm512_set1_ph_from_ps(float a) {
|
|
1481
|
-
unsigned short h = _cvtss_sh(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
1482
|
-
return (__m512h)_mm512_set1_epi16(h);
|
|
1483
|
-
}
|
|
1484
|
-
|
|
1485
|
-
SIMSIMD_PUBLIC void simsimd_sum_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1486
|
-
simsimd_f16_t *result) {
|
|
1487
|
-
__mmask32 mask = 0xFFFFFFFF;
|
|
1488
|
-
__m512h a_f16_vec, b_f16_vec;
|
|
1489
|
-
__m512h sum_f16_vec;
|
|
1490
|
-
simsimd_sum_f16_sapphire_cycle:
|
|
1491
|
-
if (n < 32) {
|
|
1492
|
-
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1493
|
-
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1494
|
-
b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1495
|
-
n = 0;
|
|
1496
|
-
}
|
|
1497
|
-
else {
|
|
1498
|
-
a_f16_vec = _mm512_loadu_ph(a);
|
|
1499
|
-
b_f16_vec = _mm512_loadu_ph(b);
|
|
1500
|
-
a += 32, b += 32, n -= 32;
|
|
1501
|
-
}
|
|
1502
|
-
sum_f16_vec = _mm512_add_ph(a_f16_vec, b_f16_vec);
|
|
1503
|
-
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1504
|
-
result += 32;
|
|
1505
|
-
if (n) goto simsimd_sum_f16_sapphire_cycle;
|
|
1506
|
-
}
|
|
1507
|
-
|
|
1508
|
-
SIMSIMD_PUBLIC void simsimd_scale_f16_sapphire(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1509
|
-
simsimd_f16_t *result) {
|
|
1510
|
-
|
|
1511
|
-
__mmask32 mask = 0xFFFFFFFF;
|
|
1512
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1513
|
-
__m512h a_f16_vec, b_f16_vec;
|
|
1514
|
-
__m512h sum_f16_vec;
|
|
1515
|
-
simsimd_scale_f16_sapphire_cycle:
|
|
1516
|
-
if (n < 32) {
|
|
1517
|
-
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1518
|
-
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1519
|
-
n = 0;
|
|
1520
|
-
}
|
|
1521
|
-
else {
|
|
1522
|
-
a_f16_vec = _mm512_loadu_ph(a);
|
|
1523
|
-
a += 32, n -= 32;
|
|
1524
|
-
}
|
|
1525
|
-
sum_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
|
|
1526
|
-
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1527
|
-
result += 32;
|
|
1528
|
-
if (n) goto simsimd_scale_f16_sapphire_cycle;
|
|
1529
|
-
}
|
|
1530
|
-
|
|
1531
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
|
|
1532
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
1533
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
1534
|
-
|
|
1535
|
-
// There are are several special cases we may want to implement:
|
|
1536
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1537
|
-
if (alpha == 1 && beta == 1) {
|
|
1538
|
-
// In this case we can avoid expensive multiplications.
|
|
1539
|
-
simsimd_sum_f16_sapphire(a, b, n, result);
|
|
1540
|
-
return;
|
|
1541
|
-
}
|
|
1542
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1543
|
-
else if (alpha == 0 || beta == 0) {
|
|
1544
|
-
// In this case we can avoid half of the load instructions.
|
|
1545
|
-
if (beta == 0) { simsimd_scale_f16_sapphire(a, n, alpha, result); }
|
|
1546
|
-
else { simsimd_scale_f16_sapphire(b, n, beta, result); }
|
|
1547
|
-
return;
|
|
1548
|
-
}
|
|
1549
|
-
|
|
1550
|
-
// The general case.
|
|
1551
|
-
__mmask32 mask = 0xFFFFFFFF;
|
|
1552
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1553
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1554
|
-
__m512h a_f16_vec, b_f16_vec;
|
|
1555
|
-
__m512h a_scaled_f16_vec, sum_f16_vec;
|
|
1556
|
-
simsimd_wsum_f16_sapphire_cycle:
|
|
1557
|
-
if (n < 32) {
|
|
1558
|
-
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1559
|
-
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1560
|
-
b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1561
|
-
n = 0;
|
|
1562
|
-
}
|
|
1563
|
-
else {
|
|
1564
|
-
a_f16_vec = _mm512_loadu_ph(a);
|
|
1565
|
-
b_f16_vec = _mm512_loadu_ph(b);
|
|
1566
|
-
a += 32, b += 32, n -= 32;
|
|
1567
|
-
}
|
|
1568
|
-
a_scaled_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
|
|
1569
|
-
sum_f16_vec = _mm512_fmadd_ph(b_f16_vec, beta_vec, a_scaled_f16_vec);
|
|
1570
|
-
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1571
|
-
result += 32;
|
|
1572
|
-
if (n) goto simsimd_wsum_f16_sapphire_cycle;
|
|
1573
|
-
}
|
|
1574
|
-
|
|
1575
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
|
|
1576
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
|
|
1577
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
1578
|
-
|
|
1579
|
-
__mmask32 mask = 0xFFFFFFFF;
|
|
1580
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1581
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1582
|
-
__m512h a_f16_vec, b_f16_vec, c_f16_vec;
|
|
1583
|
-
__m512h ab_f16_vec, ab_scaled_f16_vec, sum_f16_vec;
|
|
1584
|
-
simsimd_fma_f16_sapphire_cycle:
|
|
1585
|
-
if (n < 32) {
|
|
1586
|
-
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1587
|
-
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1588
|
-
b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1589
|
-
c_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c));
|
|
1590
|
-
n = 0;
|
|
1591
|
-
}
|
|
1592
|
-
else {
|
|
1593
|
-
a_f16_vec = _mm512_loadu_ph(a);
|
|
1594
|
-
b_f16_vec = _mm512_loadu_ph(b);
|
|
1595
|
-
c_f16_vec = _mm512_loadu_ph(c);
|
|
1596
|
-
a += 32, b += 32, c += 32, n -= 32;
|
|
1597
|
-
}
|
|
1598
|
-
ab_f16_vec = _mm512_mul_ph(a_f16_vec, b_f16_vec);
|
|
1599
|
-
ab_scaled_f16_vec = _mm512_mul_ph(ab_f16_vec, alpha_vec);
|
|
1600
|
-
sum_f16_vec = _mm512_fmadd_ph(c_f16_vec, beta_vec, ab_scaled_f16_vec);
|
|
1601
|
-
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1602
|
-
result += 32;
|
|
1603
|
-
if (n) goto simsimd_fma_f16_sapphire_cycle;
|
|
1604
|
-
}
|
|
1605
|
-
|
|
1606
|
-
SIMSIMD_PUBLIC void simsimd_sum_u8_sapphire(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1607
|
-
simsimd_u8_t *result) {
|
|
1608
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1609
|
-
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
1610
|
-
simsimd_sum_u8_sapphire_cycle:
|
|
1611
|
-
if (n < 64) {
|
|
1612
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1613
|
-
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1614
|
-
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1615
|
-
n = 0;
|
|
1616
|
-
}
|
|
1617
|
-
else {
|
|
1618
|
-
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1619
|
-
b_u8_vec = _mm512_loadu_epi8(b);
|
|
1620
|
-
a += 64, b += 64, n -= 64;
|
|
1621
|
-
}
|
|
1622
|
-
sum_u8_vec = _mm512_adds_epu8(a_u8_vec, b_u8_vec);
|
|
1623
|
-
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1624
|
-
result += 64;
|
|
1625
|
-
if (n) goto simsimd_sum_u8_sapphire_cycle;
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
|
-
SIMSIMD_PUBLIC void simsimd_scale_u8_sapphire(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1629
|
-
simsimd_u8_t *result) {
|
|
1630
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1631
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1632
|
-
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
1633
|
-
__m512h a_f16_low_vec, a_f16_high_vec;
|
|
1634
|
-
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1635
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1636
|
-
simsimd_scale_u8_sapphire_cycle:
|
|
1637
|
-
if (n < 64) {
|
|
1638
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1639
|
-
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1640
|
-
n = 0;
|
|
1641
|
-
}
|
|
1642
|
-
else {
|
|
1643
|
-
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1644
|
-
a += 64, n -= 64;
|
|
1645
|
-
}
|
|
1646
|
-
// Upcast:
|
|
1647
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1648
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1649
|
-
// Scale:
|
|
1650
|
-
sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1651
|
-
sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1652
|
-
// Downcast:
|
|
1653
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1654
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1655
|
-
sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
|
|
1656
|
-
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1657
|
-
result += 64;
|
|
1658
|
-
if (n) goto simsimd_scale_u8_sapphire_cycle;
|
|
1659
|
-
}
|
|
1660
|
-
|
|
1661
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
|
|
1662
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
1663
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
1664
|
-
|
|
1665
|
-
// There are are several special cases we may want to implement:
|
|
1666
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1667
|
-
if (alpha == 1 && beta == 1) {
|
|
1668
|
-
// In this case we can avoid expensive multiplications.
|
|
1669
|
-
simsimd_sum_u8_sapphire(a, b, n, result);
|
|
1670
|
-
return;
|
|
1671
|
-
}
|
|
1672
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1673
|
-
else if (alpha == 0 || beta == 0) {
|
|
1674
|
-
// In this case we can avoid half of the load instructions.
|
|
1675
|
-
if (beta == 0) { simsimd_scale_u8_sapphire(a, n, alpha, result); }
|
|
1676
|
-
else { simsimd_scale_u8_sapphire(b, n, beta, result); }
|
|
1677
|
-
return;
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
// The general case.
|
|
1681
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1682
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1683
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1684
|
-
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
1685
|
-
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1686
|
-
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1687
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1688
|
-
simsimd_wsum_u8_sapphire_cycle:
|
|
1689
|
-
if (n < 64) {
|
|
1690
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1691
|
-
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1692
|
-
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1693
|
-
n = 0;
|
|
1694
|
-
}
|
|
1695
|
-
else {
|
|
1696
|
-
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1697
|
-
b_u8_vec = _mm512_loadu_epi8(b);
|
|
1698
|
-
a += 64, b += 64, n -= 64;
|
|
1699
|
-
}
|
|
1700
|
-
// Upcast:
|
|
1701
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1702
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1703
|
-
b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
|
|
1704
|
-
b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
|
|
1705
|
-
// Scale:
|
|
1706
|
-
a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1707
|
-
a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1708
|
-
// Add:
|
|
1709
|
-
sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
|
|
1710
|
-
sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
|
|
1711
|
-
// Downcast:
|
|
1712
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1713
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1714
|
-
sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
|
|
1715
|
-
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1716
|
-
result += 64;
|
|
1717
|
-
if (n) goto simsimd_wsum_u8_sapphire_cycle;
|
|
1718
|
-
}
|
|
1719
|
-
|
|
1720
|
-
SIMSIMD_PUBLIC void simsimd_sum_i8_sapphire(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1721
|
-
simsimd_i8_t *result) {
|
|
1722
|
-
|
|
1723
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1724
|
-
__m512i a_i8_vec, b_i8_vec, sum_i8_vec;
|
|
1725
|
-
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1726
|
-
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1727
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1728
|
-
|
|
1729
|
-
simsimd_sum_i8_sapphire_cycle:
|
|
1730
|
-
if (n < 64) {
|
|
1731
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1732
|
-
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1733
|
-
b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1734
|
-
n = 0;
|
|
1735
|
-
}
|
|
1736
|
-
else {
|
|
1737
|
-
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1738
|
-
b_i8_vec = _mm512_loadu_epi8(b);
|
|
1739
|
-
a += 64, b += 64, n -= 64;
|
|
1740
|
-
}
|
|
1741
|
-
sum_i8_vec = _mm512_adds_epi8(a_i8_vec, b_i8_vec);
|
|
1742
|
-
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1743
|
-
result += 64;
|
|
1744
|
-
if (n) goto simsimd_sum_i8_sapphire_cycle;
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
|
-
SIMSIMD_PUBLIC void simsimd_scale_i8_sapphire(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1748
|
-
simsimd_i8_t *result) {
|
|
1749
|
-
|
|
1750
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1751
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1752
|
-
__m512i a_i8_vec, sum_i8_vec;
|
|
1753
|
-
__m512h a_f16_low_vec, a_f16_high_vec;
|
|
1754
|
-
__m512h sum_f16_low_vec, sum_f16_high_vec;
|
|
1755
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1756
|
-
simsimd_wsum_i8_sapphire_cycle:
|
|
1757
|
-
if (n < 64) {
|
|
1758
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1759
|
-
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1760
|
-
n = 0;
|
|
1761
|
-
}
|
|
1762
|
-
else {
|
|
1763
|
-
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1764
|
-
a += 64, n -= 64;
|
|
1765
|
-
}
|
|
1766
|
-
// Upcast:
|
|
1767
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
|
|
1768
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
|
|
1769
|
-
// Scale:
|
|
1770
|
-
sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1771
|
-
sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1772
|
-
// Downcast:
|
|
1773
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1774
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1775
|
-
sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
|
|
1776
|
-
_mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
|
|
1777
|
-
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1778
|
-
result += 64;
|
|
1779
|
-
if (n) goto simsimd_wsum_i8_sapphire_cycle;
|
|
1780
|
-
}
|
|
1781
|
-
|
|
1782
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
|
|
1783
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
1784
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
1785
|
-
|
|
1786
|
-
// There are are several special cases we may want to implement:
|
|
1787
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1788
|
-
if (alpha == 1 && beta == 1) {
|
|
1789
|
-
// In this case we can avoid expensive multiplications.
|
|
1790
|
-
simsimd_sum_i8_sapphire(a, b, n, result);
|
|
1791
|
-
return;
|
|
1792
|
-
}
|
|
1793
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1794
|
-
else if (alpha == 0 || beta == 0) {
|
|
1795
|
-
// In this case we can avoid half of the load instructions.
|
|
1796
|
-
if (beta == 0) { simsimd_scale_i8_sapphire(a, n, alpha, result); }
|
|
1797
|
-
else { simsimd_scale_i8_sapphire(b, n, beta, result); }
|
|
1798
|
-
return;
|
|
1799
|
-
}
|
|
1800
|
-
|
|
1801
|
-
// The general case.
|
|
1802
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1803
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1804
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1805
|
-
__m512i a_i8_vec, b_i8_vec, sum_i8_vec;
|
|
1806
|
-
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1807
|
-
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1808
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1809
|
-
|
|
1810
|
-
simsimd_wsum_i8_sapphire_cycle:
|
|
1811
|
-
if (n < 64) {
|
|
1812
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1813
|
-
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1814
|
-
b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1815
|
-
n = 0;
|
|
1816
|
-
}
|
|
1817
|
-
else {
|
|
1818
|
-
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1819
|
-
b_i8_vec = _mm512_loadu_epi8(b);
|
|
1820
|
-
a += 64, b += 64, n -= 64;
|
|
1821
|
-
}
|
|
1822
|
-
// Upcast:
|
|
1823
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
|
|
1824
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
|
|
1825
|
-
b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
|
|
1826
|
-
b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
|
|
1827
|
-
// Scale:
|
|
1828
|
-
a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1829
|
-
a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1830
|
-
// Add:
|
|
1831
|
-
sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
|
|
1832
|
-
sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
|
|
1833
|
-
// Downcast:
|
|
1834
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1835
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1836
|
-
sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
|
|
1837
|
-
_mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
|
|
1838
|
-
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1839
|
-
result += 64;
|
|
1840
|
-
if (n) goto simsimd_wsum_i8_sapphire_cycle;
|
|
1841
|
-
}
|
|
1842
|
-
|
|
1843
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
|
|
1844
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
|
|
1845
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
1846
|
-
|
|
1847
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFF;
|
|
1848
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1849
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1850
|
-
__m512i a_i8_vec, b_i8_vec, c_i8_vec, sum_i8_vec;
|
|
1851
|
-
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1852
|
-
__m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
|
|
1853
|
-
__m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1854
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1855
|
-
|
|
1856
|
-
simsimd_fma_i8_sapphire_cycle:
|
|
1857
|
-
if (n < 64) {
|
|
1858
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1859
|
-
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1860
|
-
b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1861
|
-
c_i8_vec = _mm512_maskz_loadu_epi8(mask, c);
|
|
1862
|
-
n = 0;
|
|
1863
|
-
}
|
|
1864
|
-
else {
|
|
1865
|
-
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1866
|
-
b_i8_vec = _mm512_loadu_epi8(b);
|
|
1867
|
-
c_i8_vec = _mm512_loadu_epi8(c);
|
|
1868
|
-
a += 64, b += 64, c += 64, n -= 64;
|
|
1869
|
-
}
|
|
1870
|
-
// Upcast:
|
|
1871
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
|
|
1872
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
|
|
1873
|
-
b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
|
|
1874
|
-
b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
|
|
1875
|
-
c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(c_i8_vec)));
|
|
1876
|
-
c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(c_i8_vec, 1)));
|
|
1877
|
-
// Multiply:
|
|
1878
|
-
ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
|
|
1879
|
-
ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
|
|
1880
|
-
// Scale:
|
|
1881
|
-
ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
|
|
1882
|
-
ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
|
|
1883
|
-
// Add:
|
|
1884
|
-
sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
|
|
1885
|
-
sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
|
|
1886
|
-
// Downcast:
|
|
1887
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1888
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1889
|
-
sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
|
|
1890
|
-
_mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
|
|
1891
|
-
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1892
|
-
result += 64;
|
|
1893
|
-
if (n) goto simsimd_fma_i8_sapphire_cycle;
|
|
1894
|
-
}
|
|
1895
|
-
|
|
1896
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
|
|
1897
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
|
|
1898
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
1899
|
-
|
|
1900
|
-
__mmask64 mask = 0xFFFFFFFFFFFFFFFF;
|
|
1901
|
-
__m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
|
|
1902
|
-
__m512h beta_vec = _mm512_set1_ph_from_ps(beta);
|
|
1903
|
-
__m512i a_u8_vec, b_u8_vec, c_u8_vec, sum_u8_vec;
|
|
1904
|
-
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1905
|
-
__m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
|
|
1906
|
-
__m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1907
|
-
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1908
|
-
|
|
1909
|
-
simsimd_fma_u8_sapphire_cycle:
|
|
1910
|
-
if (n < 64) {
|
|
1911
|
-
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1912
|
-
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1913
|
-
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1914
|
-
c_u8_vec = _mm512_maskz_loadu_epi8(mask, c);
|
|
1915
|
-
n = 0;
|
|
1916
|
-
}
|
|
1917
|
-
else {
|
|
1918
|
-
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1919
|
-
b_u8_vec = _mm512_loadu_epi8(b);
|
|
1920
|
-
c_u8_vec = _mm512_loadu_epi8(c);
|
|
1921
|
-
a += 64, b += 64, c += 64, n -= 64;
|
|
1922
|
-
}
|
|
1923
|
-
// Upcast:
|
|
1924
|
-
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1925
|
-
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1926
|
-
b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
|
|
1927
|
-
b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
|
|
1928
|
-
c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(c_u8_vec, _mm512_setzero_si512()));
|
|
1929
|
-
c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(c_u8_vec, _mm512_setzero_si512()));
|
|
1930
|
-
// Multiply:
|
|
1931
|
-
ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
|
|
1932
|
-
ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
|
|
1933
|
-
// Scale:
|
|
1934
|
-
ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
|
|
1935
|
-
ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
|
|
1936
|
-
// Add:
|
|
1937
|
-
sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
|
|
1938
|
-
sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
|
|
1939
|
-
// Downcast:
|
|
1940
|
-
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1941
|
-
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1942
|
-
sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
|
|
1943
|
-
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1944
|
-
result += 64;
|
|
1945
|
-
if (n) goto simsimd_fma_u8_sapphire_cycle;
|
|
1946
|
-
}
|
|
1947
|
-
|
|
1948
|
-
#pragma clang attribute pop
|
|
1949
|
-
#pragma GCC pop_options
|
|
1950
|
-
#endif // SIMSIMD_TARGET_SAPPHIRE
|
|
1951
|
-
#endif // _SIMSIMD_TARGET_X86
|
|
1952
|
-
|
|
1953
|
-
#if _SIMSIMD_TARGET_ARM
|
|
1954
|
-
#if SIMSIMD_TARGET_NEON
|
|
1955
|
-
#pragma GCC push_options
|
|
1956
|
-
#pragma GCC target("arch=armv8.2-a+simd")
|
|
1957
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
1958
|
-
|
|
1959
|
-
SIMSIMD_PUBLIC void simsimd_sum_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1960
|
-
simsimd_f32_t *result) {
|
|
1961
|
-
// The main loop:
|
|
1962
|
-
simsimd_size_t i = 0;
|
|
1963
|
-
for (; i + 4 <= n; i += 4) {
|
|
1964
|
-
float32x4_t a_vec = vld1q_f32(a + i);
|
|
1965
|
-
float32x4_t b_vec = vld1q_f32(b + i);
|
|
1966
|
-
float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
|
|
1967
|
-
vst1q_f32(result + i, sum_vec);
|
|
1968
|
-
}
|
|
1969
|
-
|
|
1970
|
-
// The tail:
|
|
1971
|
-
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
1972
|
-
}
|
|
1973
|
-
|
|
1974
|
-
SIMSIMD_PUBLIC void simsimd_scale_f32_neon(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1975
|
-
simsimd_f32_t *result) {
|
|
1976
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
1977
|
-
|
|
1978
|
-
// The main loop:
|
|
1979
|
-
simsimd_size_t i = 0;
|
|
1980
|
-
for (; i + 4 <= n; i += 4) {
|
|
1981
|
-
float32x4_t a_vec = vld1q_f32(a + i);
|
|
1982
|
-
float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
1983
|
-
vst1q_f32(result + i, sum_vec);
|
|
1984
|
-
}
|
|
1985
|
-
|
|
1986
|
-
// The tail:
|
|
1987
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i];
|
|
1988
|
-
}
|
|
1989
|
-
|
|
1990
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
|
|
1991
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
1992
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
1993
|
-
|
|
1994
|
-
// There are are several special cases we may want to implement:
|
|
1995
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1996
|
-
if (alpha == 1 && beta == 1) {
|
|
1997
|
-
// In this case we can avoid expensive multiplications.
|
|
1998
|
-
simsimd_sum_f32_neon(a, b, n, result);
|
|
1999
|
-
return;
|
|
2000
|
-
}
|
|
2001
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2002
|
-
else if (alpha == 0 || beta == 0) {
|
|
2003
|
-
// In this case we can avoid half of the load instructions.
|
|
2004
|
-
if (beta == 0) { simsimd_scale_f32_neon(a, n, alpha, result); }
|
|
2005
|
-
else { simsimd_scale_f32_neon(b, n, beta, result); }
|
|
2006
|
-
return;
|
|
2007
|
-
}
|
|
2008
|
-
|
|
2009
|
-
// The general case.
|
|
2010
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2011
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
2012
|
-
|
|
2013
|
-
// The main loop:
|
|
2014
|
-
simsimd_size_t i = 0;
|
|
2015
|
-
for (; i + 4 <= n; i += 4) {
|
|
2016
|
-
float32x4_t a_vec = vld1q_f32(a + i);
|
|
2017
|
-
float32x4_t b_vec = vld1q_f32(b + i);
|
|
2018
|
-
float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
2019
|
-
float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
|
|
2020
|
-
float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
|
|
2021
|
-
vst1q_f32(result + i, sum_vec);
|
|
2022
|
-
}
|
|
2023
|
-
|
|
2024
|
-
// The tail:
|
|
2025
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
|
|
2026
|
-
}
|
|
2027
|
-
|
|
2028
|
-
SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
|
|
2029
|
-
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
|
|
2030
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
2031
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2032
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
2033
|
-
|
|
2034
|
-
// The main loop:
|
|
2035
|
-
simsimd_size_t i = 0;
|
|
2036
|
-
for (; i + 4 <= n; i += 4) {
|
|
2037
|
-
float32x4_t a_vec = vld1q_f32(a + i);
|
|
2038
|
-
float32x4_t b_vec = vld1q_f32(b + i);
|
|
2039
|
-
float32x4_t c_vec = vld1q_f32(c + i);
|
|
2040
|
-
float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
|
|
2041
|
-
float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
|
|
2042
|
-
float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
|
|
2043
|
-
vst1q_f32(result + i, sum_vec);
|
|
2044
|
-
}
|
|
2045
|
-
|
|
2046
|
-
// The tail:
|
|
2047
|
-
for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
|
|
2048
|
-
}
|
|
2049
|
-
|
|
2050
|
-
#pragma clang attribute pop
|
|
2051
|
-
#pragma GCC pop_options
|
|
2052
|
-
#endif // SIMSIMD_TARGET_NEON
|
|
2053
|
-
|
|
2054
|
-
#if SIMSIMD_TARGET_NEON_BF16
|
|
2055
|
-
#pragma GCC push_options
|
|
2056
|
-
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
2057
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
2058
|
-
|
|
2059
|
-
SIMSIMD_PUBLIC void simsimd_sum_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
2060
|
-
simsimd_bf16_t *result) {
|
|
2061
|
-
// The main loop:
|
|
2062
|
-
simsimd_size_t i = 0;
|
|
2063
|
-
for (; i + 4 <= n; i += 4) {
|
|
2064
|
-
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2065
|
-
float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
|
|
2066
|
-
float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
|
|
2067
|
-
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
// The tail:
|
|
2071
|
-
for (; i < n; ++i) simsimd_f32_to_bf16(simsimd_bf16_to_f32(a + i) + simsimd_bf16_to_f32(b + i), result + i);
|
|
2072
|
-
}
|
|
2073
|
-
|
|
2074
|
-
SIMSIMD_PUBLIC void simsimd_scale_bf16_neon(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2075
|
-
simsimd_bf16_t *result) {
|
|
2076
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2077
|
-
|
|
2078
|
-
// The main loop:
|
|
2079
|
-
simsimd_size_t i = 0;
|
|
2080
|
-
for (; i + 4 <= n; i += 4) {
|
|
2081
|
-
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2082
|
-
float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
2083
|
-
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2084
|
-
}
|
|
2085
|
-
|
|
2086
|
-
// The tail:
|
|
2087
|
-
for (; i < n; ++i) simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i), result + i);
|
|
2088
|
-
}
|
|
2089
|
-
|
|
2090
|
-
SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
|
|
2091
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
2092
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
2093
|
-
|
|
2094
|
-
// There are are several special cases we may want to implement:
|
|
2095
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2096
|
-
if (alpha == 1 && beta == 1) {
|
|
2097
|
-
// In this case we can avoid expensive multiplications.
|
|
2098
|
-
simsimd_sum_bf16_neon(a, b, n, result);
|
|
2099
|
-
return;
|
|
2100
|
-
}
|
|
2101
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2102
|
-
else if (alpha == 0 || beta == 0) {
|
|
2103
|
-
// In this case we can avoid half of the load instructions.
|
|
2104
|
-
if (beta == 0) { simsimd_scale_bf16_neon(a, n, alpha, result); }
|
|
2105
|
-
else { simsimd_scale_bf16_neon(b, n, beta, result); }
|
|
2106
|
-
return;
|
|
2107
|
-
}
|
|
2108
|
-
|
|
2109
|
-
// The general case.
|
|
2110
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2111
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
2112
|
-
|
|
2113
|
-
// The main loop:
|
|
2114
|
-
simsimd_size_t i = 0;
|
|
2115
|
-
for (; i + 4 <= n; i += 4) {
|
|
2116
|
-
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2117
|
-
float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
|
|
2118
|
-
float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
2119
|
-
float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
|
|
2120
|
-
float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
|
|
2121
|
-
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2122
|
-
}
|
|
2123
|
-
|
|
2124
|
-
// The tail:
|
|
2125
|
-
for (; i < n; ++i)
|
|
2126
|
-
simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i) + beta_f32 * simsimd_bf16_to_f32(b + i), result + i);
|
|
2127
|
-
}
|
|
2128
|
-
|
|
2129
|
-
SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
|
|
2130
|
-
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
|
|
2131
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
2132
|
-
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2133
|
-
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
2134
|
-
|
|
2135
|
-
// The main loop:
|
|
2136
|
-
simsimd_size_t i = 0;
|
|
2137
|
-
for (; i + 4 <= n; i += 4) {
|
|
2138
|
-
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2139
|
-
float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
|
|
2140
|
-
float32x4_t c_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)c + i));
|
|
2141
|
-
float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
|
|
2142
|
-
float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
|
|
2143
|
-
float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
|
|
2144
|
-
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2145
|
-
}
|
|
2146
|
-
|
|
2147
|
-
// The tail:
|
|
2148
|
-
for (; i < n; ++i)
|
|
2149
|
-
simsimd_f32_to_bf16(
|
|
2150
|
-
alpha_f32 * simsimd_bf16_to_f32(a + i) * simsimd_bf16_to_f32(b + i) + beta_f32 * simsimd_bf16_to_f32(c + i),
|
|
2151
|
-
result + i);
|
|
2152
|
-
}
|
|
2153
|
-
|
|
2154
|
-
#pragma clang attribute pop
|
|
2155
|
-
#pragma GCC pop_options
|
|
2156
|
-
#endif // SIMSIMD_TARGET_NEON_BF16
|
|
2157
|
-
|
|
2158
|
-
#if SIMSIMD_TARGET_NEON_F16
|
|
2159
|
-
#pragma GCC push_options
|
|
2160
|
-
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
2161
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
2162
|
-
|
|
2163
|
-
SIMSIMD_PUBLIC void simsimd_sum_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
2164
|
-
simsimd_f16_t *result) {
|
|
2165
|
-
// The main loop:
|
|
2166
|
-
simsimd_size_t i = 0;
|
|
2167
|
-
for (; i + 8 <= n; i += 8) {
|
|
2168
|
-
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2169
|
-
float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
|
|
2170
|
-
float16x8_t sum_vec = vaddq_f16(a_vec, b_vec);
|
|
2171
|
-
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2172
|
-
}
|
|
2173
|
-
|
|
2174
|
-
// The tail:
|
|
2175
|
-
for (; i < n; ++i) ((float16_t *)result)[i] = ((float16_t const *)a)[i] + ((float16_t const *)b)[i];
|
|
2176
|
-
}
|
|
2177
|
-
|
|
2178
|
-
SIMSIMD_PUBLIC void simsimd_scale_f16_neon(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2179
|
-
simsimd_f16_t *result) {
|
|
2180
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2181
|
-
|
|
2182
|
-
// The main loop:
|
|
2183
|
-
simsimd_size_t i = 0;
|
|
2184
|
-
for (; i + 8 <= n; i += 8) {
|
|
2185
|
-
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2186
|
-
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2187
|
-
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2188
|
-
}
|
|
2189
|
-
|
|
2190
|
-
// The tail:
|
|
2191
|
-
for (; i < n; ++i) ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i];
|
|
2192
|
-
}
|
|
2193
|
-
|
|
2194
|
-
SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
|
|
2195
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
2196
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
2197
|
-
// There are are several special cases we may want to implement:
|
|
2198
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2199
|
-
if (alpha == 1 && beta == 1) {
|
|
2200
|
-
// In this case we can avoid expensive multiplications.
|
|
2201
|
-
simsimd_sum_f16_neon(a, b, n, result);
|
|
2202
|
-
return;
|
|
2203
|
-
}
|
|
2204
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2205
|
-
else if (alpha == 0 || beta == 0) {
|
|
2206
|
-
// In this case we can avoid half of the load instructions.
|
|
2207
|
-
if (beta == 0) { simsimd_scale_f16_neon(a, n, alpha, result); }
|
|
2208
|
-
else { simsimd_scale_f16_neon(b, n, beta, result); }
|
|
2209
|
-
return;
|
|
2210
|
-
}
|
|
2211
|
-
|
|
2212
|
-
// The general case.
|
|
2213
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2214
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2215
|
-
|
|
2216
|
-
// The main loop:
|
|
2217
|
-
simsimd_size_t i = 0;
|
|
2218
|
-
for (; i + 8 <= n; i += 8) {
|
|
2219
|
-
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2220
|
-
float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
|
|
2221
|
-
float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2222
|
-
float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
|
|
2223
|
-
float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
|
|
2224
|
-
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2225
|
-
}
|
|
2226
|
-
|
|
2227
|
-
// The tail:
|
|
2228
|
-
for (; i < n; ++i)
|
|
2229
|
-
((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i] + beta_f16 * ((float16_t const *)b)[i];
|
|
2230
|
-
}
|
|
2231
|
-
|
|
2232
|
-
SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
|
|
2233
|
-
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
|
|
2234
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
2235
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2236
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2237
|
-
|
|
2238
|
-
// The main loop:
|
|
2239
|
-
simsimd_size_t i = 0;
|
|
2240
|
-
for (; i + 8 <= n; i += 8) {
|
|
2241
|
-
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2242
|
-
float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
|
|
2243
|
-
float16x8_t c_vec = vld1q_f16((float16_t const *)c + i);
|
|
2244
|
-
float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
|
|
2245
|
-
float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
|
|
2246
|
-
float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
|
|
2247
|
-
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2248
|
-
}
|
|
2249
|
-
|
|
2250
|
-
// The tail:
|
|
2251
|
-
for (; i < n; ++i)
|
|
2252
|
-
((float16_t *)result)[i] =
|
|
2253
|
-
alpha_f16 * ((float16_t const *)a)[i] * ((float16_t const *)b)[i] + beta_f16 * ((float16_t const *)c)[i];
|
|
2254
|
-
}
|
|
2255
|
-
|
|
2256
|
-
SIMSIMD_PUBLIC void simsimd_sum_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
2257
|
-
simsimd_u8_t *result) {
|
|
2258
|
-
// The main loop:
|
|
2259
|
-
simsimd_size_t i = 0;
|
|
2260
|
-
for (; i + 16 <= n; i += 16) {
|
|
2261
|
-
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
2262
|
-
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
2263
|
-
uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
|
|
2264
|
-
vst1q_u8(result + i, sum_vec);
|
|
2265
|
-
}
|
|
2266
|
-
|
|
2267
|
-
// The tail:
|
|
2268
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(a[i] + b[i], result + i); }
|
|
2269
|
-
}
|
|
2270
|
-
|
|
2271
|
-
SIMSIMD_PUBLIC void simsimd_scale_u8_neon(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2272
|
-
simsimd_u8_t *result) {
|
|
2273
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2274
|
-
|
|
2275
|
-
// The main loop:
|
|
2276
|
-
simsimd_size_t i = 0;
|
|
2277
|
-
for (; i + 8 <= n; i += 8) {
|
|
2278
|
-
uint8x8_t a_u8_vec = vld1_u8(a + i);
|
|
2279
|
-
float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
|
|
2280
|
-
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2281
|
-
uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
|
|
2282
|
-
vst1_u8(result + i, sum_u8_vec);
|
|
2283
|
-
}
|
|
2284
|
-
|
|
2285
|
-
// The tail:
|
|
2286
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i], result + i); }
|
|
2287
|
-
}
|
|
2288
|
-
|
|
2289
|
-
SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
|
|
2290
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
2291
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
2292
|
-
|
|
2293
|
-
// There are are several special cases we may want to implement:
|
|
2294
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2295
|
-
if (alpha == 1 && beta == 1) {
|
|
2296
|
-
// In this case we can avoid expensive multiplications.
|
|
2297
|
-
simsimd_sum_u8_neon(a, b, n, result);
|
|
2298
|
-
return;
|
|
2299
|
-
}
|
|
2300
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2301
|
-
else if (alpha == 0 || beta == 0) {
|
|
2302
|
-
// In this case we can avoid half of the load instructions.
|
|
2303
|
-
if (beta == 0) { simsimd_scale_u8_neon(a, n, alpha, result); }
|
|
2304
|
-
else { simsimd_scale_u8_neon(b, n, beta, result); }
|
|
2305
|
-
return;
|
|
2306
|
-
}
|
|
2307
|
-
|
|
2308
|
-
// The general case.
|
|
2309
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2310
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2311
|
-
|
|
2312
|
-
// The main loop:
|
|
2313
|
-
simsimd_size_t i = 0;
|
|
2314
|
-
for (; i + 8 <= n; i += 8) {
|
|
2315
|
-
uint8x8_t a_u8_vec = vld1_u8(a + i);
|
|
2316
|
-
uint8x8_t b_u8_vec = vld1_u8(b + i);
|
|
2317
|
-
float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
|
|
2318
|
-
float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
|
|
2319
|
-
float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2320
|
-
float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
|
|
2321
|
-
float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
|
|
2322
|
-
uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
|
|
2323
|
-
vst1_u8(result + i, sum_u8_vec);
|
|
2324
|
-
}
|
|
2325
|
-
|
|
2326
|
-
// The tail:
|
|
2327
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
|
|
2328
|
-
}
|
|
2329
|
-
|
|
2330
|
-
SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
|
|
2331
|
-
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
|
|
2332
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
2333
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2334
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2335
|
-
|
|
2336
|
-
// The main loop:
|
|
2337
|
-
simsimd_size_t i = 0;
|
|
2338
|
-
for (; i + 8 <= n; i += 8) {
|
|
2339
|
-
uint8x8_t a_u8_vec = vld1_u8(a + i);
|
|
2340
|
-
uint8x8_t b_u8_vec = vld1_u8(b + i);
|
|
2341
|
-
uint8x8_t c_u8_vec = vld1_u8(c + i);
|
|
2342
|
-
float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
|
|
2343
|
-
float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
|
|
2344
|
-
float16x8_t c_vec = vcvtq_f16_u16(vmovl_u8(c_u8_vec));
|
|
2345
|
-
float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
|
|
2346
|
-
float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
|
|
2347
|
-
float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
|
|
2348
|
-
uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
|
|
2349
|
-
vst1_u8(result + i, sum_u8_vec);
|
|
2350
|
-
}
|
|
2351
|
-
|
|
2352
|
-
// The tail:
|
|
2353
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
|
|
2354
|
-
}
|
|
2355
|
-
|
|
2356
|
-
SIMSIMD_PUBLIC void simsimd_sum_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
2357
|
-
simsimd_i8_t *result) {
|
|
2358
|
-
// The main loop:
|
|
2359
|
-
simsimd_size_t i = 0;
|
|
2360
|
-
for (; i + 16 <= n; i += 16) {
|
|
2361
|
-
int8x16_t a_vec = vld1q_s8(a + i);
|
|
2362
|
-
int8x16_t b_vec = vld1q_s8(b + i);
|
|
2363
|
-
int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
|
|
2364
|
-
vst1q_s8(result + i, sum_vec);
|
|
2365
|
-
}
|
|
2366
|
-
|
|
2367
|
-
// The tail:
|
|
2368
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(a[i] + b[i], result + i); }
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
SIMSIMD_PUBLIC void simsimd_scale_i8_neon(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2372
|
-
simsimd_i8_t *result) {
|
|
2373
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2374
|
-
|
|
2375
|
-
// The main loop:
|
|
2376
|
-
simsimd_size_t i = 0;
|
|
2377
|
-
for (; i + 8 <= n; i += 8) {
|
|
2378
|
-
int8x8_t a_i8_vec = vld1_s8(a + i);
|
|
2379
|
-
float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
|
|
2380
|
-
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2381
|
-
int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
|
|
2382
|
-
vst1_s8(result + i, sum_i8_vec);
|
|
2383
|
-
}
|
|
2384
|
-
|
|
2385
|
-
// The tail:
|
|
2386
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i], result + i); }
|
|
2387
|
-
}
|
|
2388
|
-
|
|
2389
|
-
SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
|
|
2390
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
2391
|
-
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
2392
|
-
|
|
2393
|
-
// There are are several special cases we may want to implement:
|
|
2394
|
-
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2395
|
-
if (alpha == 1 && beta == 1) {
|
|
2396
|
-
// In this case we can avoid expensive multiplications.
|
|
2397
|
-
simsimd_sum_i8_neon(a, b, n, result);
|
|
2398
|
-
return;
|
|
2399
|
-
}
|
|
2400
|
-
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2401
|
-
else if (alpha == 0 || beta == 0) {
|
|
2402
|
-
// In this case we can avoid half of the load instructions.
|
|
2403
|
-
if (beta == 0) { simsimd_scale_i8_neon(a, n, alpha, result); }
|
|
2404
|
-
else { simsimd_scale_i8_neon(b, n, beta, result); }
|
|
2405
|
-
return;
|
|
2406
|
-
}
|
|
2407
|
-
|
|
2408
|
-
// The general case.
|
|
2409
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2410
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2411
|
-
|
|
2412
|
-
// The main loop:
|
|
2413
|
-
simsimd_size_t i = 0;
|
|
2414
|
-
for (; i + 8 <= n; i += 8) {
|
|
2415
|
-
int8x8_t a_i8_vec = vld1_s8(a + i);
|
|
2416
|
-
int8x8_t b_i8_vec = vld1_s8(b + i);
|
|
2417
|
-
float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
|
|
2418
|
-
float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
|
|
2419
|
-
float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2420
|
-
float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
|
|
2421
|
-
float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
|
|
2422
|
-
int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
|
|
2423
|
-
vst1_s8(result + i, sum_i8_vec);
|
|
2424
|
-
}
|
|
2425
|
-
|
|
2426
|
-
// The tail:
|
|
2427
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
|
|
2428
|
-
}
|
|
2429
|
-
|
|
2430
|
-
SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
|
|
2431
|
-
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
|
|
2432
|
-
simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
2433
|
-
float16_t alpha_f16 = (float16_t)alpha;
|
|
2434
|
-
float16_t beta_f16 = (float16_t)beta;
|
|
2435
|
-
|
|
2436
|
-
// The main loop:
|
|
2437
|
-
simsimd_size_t i = 0;
|
|
2438
|
-
for (; i + 8 <= n; i += 8) {
|
|
2439
|
-
int8x8_t a_i8_vec = vld1_s8(a + i);
|
|
2440
|
-
int8x8_t b_i8_vec = vld1_s8(b + i);
|
|
2441
|
-
int8x8_t c_i8_vec = vld1_s8(c + i);
|
|
2442
|
-
float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
|
|
2443
|
-
float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
|
|
2444
|
-
float16x8_t c_vec = vcvtq_f16_s16(vmovl_s8(c_i8_vec));
|
|
2445
|
-
float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
|
|
2446
|
-
float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
|
|
2447
|
-
float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
|
|
2448
|
-
int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
|
|
2449
|
-
vst1_s8(result + i, sum_i8_vec);
|
|
2450
|
-
}
|
|
2451
|
-
|
|
2452
|
-
// The tail:
|
|
2453
|
-
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
#pragma clang attribute pop
|
|
2457
|
-
#pragma GCC pop_options
|
|
2458
|
-
#endif // SIMSIMD_TARGET_NEON_F16
|
|
2459
|
-
#endif // _SIMSIMD_TARGET_ARM
|
|
2460
|
-
|
|
2461
|
-
#ifdef __cplusplus
|
|
2462
|
-
}
|
|
2463
|
-
#endif
|
|
2464
|
-
|
|
2465
|
-
#endif
|