usearch 2.21.4 → 2.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -69
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +8 -8
- package/c/lib.cpp +32 -14
- package/c/test.c +124 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +224 -48
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/README.md +1 -1
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.test.js +117 -94
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -1,1541 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file curved.h
|
|
3
|
-
* @brief SIMD-accelerated Similarity Measures for curved spaces.
|
|
4
|
-
* @author Ash Vardanian
|
|
5
|
-
* @date August 27, 2024
|
|
6
|
-
*
|
|
7
|
-
* Contains:
|
|
8
|
-
* - Mahalanobis distance
|
|
9
|
-
* - Bilinear form multiplication
|
|
10
|
-
* - Bilinear form multiplication over complex numbers
|
|
11
|
-
*
|
|
12
|
-
* For datatypes:
|
|
13
|
-
* - 64-bit floating point numbers
|
|
14
|
-
* - 32-bit floating point numbers
|
|
15
|
-
* - 16-bit floating point numbers
|
|
16
|
-
* - 16-bit brain-floating point numbers
|
|
17
|
-
*
|
|
18
|
-
* For hardware architectures:
|
|
19
|
-
* - Arm: NEON
|
|
20
|
-
* - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
|
|
21
|
-
*
|
|
22
|
-
* Most kernels in this file are designed for BLAS level 2 operations, where the operands are
|
|
23
|
-
* a combination of matrices and vectors, generally forming a chain of multiplications.
|
|
24
|
-
* Most kernels exploit the fact that matrix multiplication is associative, and the order of
|
|
25
|
-
* operations can be changed to minimize the number of operations: `(A * B) * C = A * (B * C)`.
|
|
26
|
-
* To optimize the performance, we minimize the number of memory accesses, and maximize the
|
|
27
|
-
* number of arithmetic operations, by using SIMD instructions.
|
|
28
|
-
*
|
|
29
|
-
* When A and C are vectors, and B is a matrix, we can load every element in B just once, and
|
|
30
|
-
* reuse it for every element in A and C.
|
|
31
|
-
*
|
|
32
|
-
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
33
|
-
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
34
|
-
*/
|
|
35
|
-
#ifndef SIMSIMD_CURVED_H
|
|
36
|
-
#define SIMSIMD_CURVED_H
|
|
37
|
-
|
|
38
|
-
#include "types.h"
|
|
39
|
-
|
|
40
|
-
#include "dot.h" // `_simsimd_partial_load_f16x4_neon` and friends
|
|
41
|
-
#include "spatial.h" // `_simsimd_substract_bf16x32_genoa`
|
|
42
|
-
|
|
43
|
-
#ifdef __cplusplus
|
|
44
|
-
extern "C" {
|
|
45
|
-
#endif
|
|
46
|
-
|
|
47
|
-
// clang-format off
|
|
48
|
-
|
|
49
|
-
/* Serial backends for all numeric types.
|
|
50
|
-
* By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
|
|
51
|
-
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
52
|
-
*/
|
|
53
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
54
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
55
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
56
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
57
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
58
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
59
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
60
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
61
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
62
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
63
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
64
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
65
|
-
|
|
66
|
-
/* Double-precision serial backends for all numeric types.
|
|
67
|
-
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
68
|
-
*/
|
|
69
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
70
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
71
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
72
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
73
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
74
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
75
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
76
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
77
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
78
|
-
|
|
79
|
-
/* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
|
|
80
|
-
* By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
|
|
81
|
-
* server CPUs produced before 2023.
|
|
82
|
-
*/
|
|
83
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
84
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
85
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
86
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
87
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
88
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
89
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
90
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
91
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
92
|
-
|
|
93
|
-
/* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
|
|
94
|
-
* First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
|
|
95
|
-
* Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
|
|
96
|
-
* On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
|
|
97
|
-
* properly vectorized by recent compilers.
|
|
98
|
-
*/
|
|
99
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
100
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
101
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
102
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
103
|
-
|
|
104
|
-
/* SIMD-powered backends for various generations of AVX512 CPUs.
|
|
105
|
-
* Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
|
|
106
|
-
* Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
|
|
107
|
-
* Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
|
|
108
|
-
*/
|
|
109
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
110
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
111
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
112
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
113
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
114
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
115
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
116
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
117
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
118
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
119
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
120
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
121
|
-
// clang-format on
|
|
122
|
-
|
|
123
|
-
#define SIMSIMD_MAKE_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
|
|
124
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
|
|
125
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
|
|
126
|
-
simsimd_size_t n, simsimd_distance_t *result) { \
|
|
127
|
-
simsimd_##accumulator_type##_t sum = 0; \
|
|
128
|
-
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
129
|
-
simsimd_##accumulator_type##_t cb_j = 0; \
|
|
130
|
-
simsimd_##accumulator_type##_t a_i = load_and_convert(a + i); \
|
|
131
|
-
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
132
|
-
simsimd_##accumulator_type##_t b_j = load_and_convert(b + j); \
|
|
133
|
-
simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
|
|
134
|
-
cb_j += c_ij * b_j; \
|
|
135
|
-
} \
|
|
136
|
-
sum += a_i * cb_j; \
|
|
137
|
-
} \
|
|
138
|
-
*result = (simsimd_distance_t)sum; \
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
#define SIMSIMD_MAKE_COMPLEX_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
|
|
142
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
|
|
143
|
-
simsimd_##input_type##_t const *a_pairs, simsimd_##input_type##_t const *b_pairs, \
|
|
144
|
-
simsimd_##input_type##_t const *c_pairs, simsimd_size_t n, simsimd_distance_t *results) { \
|
|
145
|
-
simsimd_##accumulator_type##_t sum_real = 0; \
|
|
146
|
-
simsimd_##accumulator_type##_t sum_imag = 0; \
|
|
147
|
-
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
148
|
-
simsimd_##accumulator_type##_t cb_j_real = 0; \
|
|
149
|
-
simsimd_##accumulator_type##_t cb_j_imag = 0; \
|
|
150
|
-
simsimd_##accumulator_type##_t a_i_real = load_and_convert(&(a_pairs + i)->real); \
|
|
151
|
-
simsimd_##accumulator_type##_t a_i_imag = load_and_convert(&(a_pairs + i)->imag); \
|
|
152
|
-
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
153
|
-
simsimd_##accumulator_type##_t b_j_real = load_and_convert(&(b_pairs + j)->real); \
|
|
154
|
-
simsimd_##accumulator_type##_t b_j_imag = load_and_convert(&(b_pairs + j)->imag); \
|
|
155
|
-
simsimd_##accumulator_type##_t c_ij_real = load_and_convert(&(c_pairs + i * n + j)->real); \
|
|
156
|
-
simsimd_##accumulator_type##_t c_ij_imag = load_and_convert(&(c_pairs + i * n + j)->imag); \
|
|
157
|
-
/* Complex multiplication: (c_ij * b_j) */ \
|
|
158
|
-
cb_j_real += c_ij_real * b_j_real - c_ij_imag * b_j_imag; \
|
|
159
|
-
cb_j_imag += c_ij_real * b_j_imag + c_ij_imag * b_j_real; \
|
|
160
|
-
} \
|
|
161
|
-
/* Complex multiplication: (a_i * cb_j) */ \
|
|
162
|
-
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag; \
|
|
163
|
-
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real; \
|
|
164
|
-
} \
|
|
165
|
-
results[0] = (simsimd_distance_t)sum_real; \
|
|
166
|
-
results[1] = (simsimd_distance_t)sum_imag; \
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
#define SIMSIMD_MAKE_MAHALANOBIS(name, input_type, accumulator_type, load_and_convert) \
|
|
170
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_##input_type##_##name( \
|
|
171
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
|
|
172
|
-
simsimd_size_t n, simsimd_distance_t *result) { \
|
|
173
|
-
simsimd_##accumulator_type##_t sum = 0; \
|
|
174
|
-
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
175
|
-
simsimd_##accumulator_type##_t cdiff_j = 0; \
|
|
176
|
-
simsimd_##accumulator_type##_t diff_i = load_and_convert(a + i) - load_and_convert(b + i); \
|
|
177
|
-
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
178
|
-
simsimd_##accumulator_type##_t diff_j = load_and_convert(a + j) - load_and_convert(b + j); \
|
|
179
|
-
simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
|
|
180
|
-
cdiff_j += c_ij * diff_j; \
|
|
181
|
-
} \
|
|
182
|
-
sum += diff_i * cdiff_j; \
|
|
183
|
-
} \
|
|
184
|
-
*result = (simsimd_distance_t)SIMSIMD_SQRT(sum); \
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64_serial
|
|
188
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64c_serial
|
|
189
|
-
SIMSIMD_MAKE_MAHALANOBIS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f64_serial
|
|
190
|
-
|
|
191
|
-
SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_serial
|
|
192
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_serial
|
|
193
|
-
SIMSIMD_MAKE_MAHALANOBIS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_serial
|
|
194
|
-
|
|
195
|
-
SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_serial
|
|
196
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_serial
|
|
197
|
-
SIMSIMD_MAKE_MAHALANOBIS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_serial
|
|
198
|
-
|
|
199
|
-
SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_serial
|
|
200
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_serial
|
|
201
|
-
SIMSIMD_MAKE_MAHALANOBIS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_serial
|
|
202
|
-
|
|
203
|
-
SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_accurate
|
|
204
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_accurate
|
|
205
|
-
SIMSIMD_MAKE_MAHALANOBIS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_accurate
|
|
206
|
-
|
|
207
|
-
SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_accurate
|
|
208
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_accurate
|
|
209
|
-
SIMSIMD_MAKE_MAHALANOBIS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_accurate
|
|
210
|
-
|
|
211
|
-
SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_accurate
|
|
212
|
-
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_accurate
|
|
213
|
-
SIMSIMD_MAKE_MAHALANOBIS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_accurate
|
|
214
|
-
|
|
215
|
-
#if _SIMSIMD_TARGET_ARM
|
|
216
|
-
#if SIMSIMD_TARGET_NEON
|
|
217
|
-
#pragma GCC push_options
|
|
218
|
-
#pragma GCC target("arch=armv8.2-a+simd")
|
|
219
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
220
|
-
|
|
221
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
|
|
222
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
223
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
224
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
225
|
-
float32x4_t a_vec = vdupq_n_f32(a[i]);
|
|
226
|
-
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
227
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
228
|
-
float32x4_t b_vec = vld1q_f32(b + j);
|
|
229
|
-
float32x4_t c_vec = vld1q_f32(c + i * n + j);
|
|
230
|
-
cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
|
|
231
|
-
}
|
|
232
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// Handle the tail of every row
|
|
236
|
-
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
237
|
-
simsimd_size_t const tail_length = n % 4;
|
|
238
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
239
|
-
if (tail_length) {
|
|
240
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
241
|
-
simsimd_f32_t a_i = a[i];
|
|
242
|
-
simsimd_f32_t cb_j = 0;
|
|
243
|
-
for (simsimd_size_t j = tail_start; j != n; ++j) cb_j += b[j] * c[i * n + j];
|
|
244
|
-
sum += a[i] * cb_j;
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
*result = sum;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
|
|
252
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
253
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
254
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
255
|
-
float32x4_t diff_i_vec = vdupq_n_f32(a[i] - b[i]);
|
|
256
|
-
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
257
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
258
|
-
float32x4_t diff_j_vec = vsubq_f32(vld1q_f32(a + j), vld1q_f32(b + j));
|
|
259
|
-
float32x4_t c_vec = vld1q_f32(c + i * n + j);
|
|
260
|
-
cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
// Handle the tail of every row
|
|
267
|
-
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
268
|
-
simsimd_size_t const tail_length = n % 4;
|
|
269
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
270
|
-
if (tail_length) {
|
|
271
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
272
|
-
simsimd_f32_t diff_i = a[i] - b[i];
|
|
273
|
-
simsimd_f32_t cdiff_j = 0;
|
|
274
|
-
for (simsimd_size_t j = tail_start; j != n; ++j) {
|
|
275
|
-
simsimd_f32_t diff_j = a[j] - b[j];
|
|
276
|
-
cdiff_j += diff_j * c[i * n + j];
|
|
277
|
-
}
|
|
278
|
-
sum += diff_i * cdiff_j;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
*result = _simsimd_sqrt_f64_neon(sum);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
|
|
286
|
-
simsimd_f32c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
|
|
287
|
-
simsimd_f32_t sum_real = 0;
|
|
288
|
-
simsimd_f32_t sum_imag = 0;
|
|
289
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
290
|
-
simsimd_f32c_t a_i = a[i];
|
|
291
|
-
simsimd_f32c_t cb_j;
|
|
292
|
-
float32x4_t cb_j_real_vec = vdupq_n_f32(0);
|
|
293
|
-
float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
|
|
294
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
295
|
-
// Unpack the input arrays into real and imaginary parts:
|
|
296
|
-
float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b + j));
|
|
297
|
-
float32x4x2_t c_vec = vld2q_f32((simsimd_f32_t const *)(c + i * n + j));
|
|
298
|
-
float32x4_t b_real_vec = b_vec.val[0];
|
|
299
|
-
float32x4_t b_imag_vec = b_vec.val[1];
|
|
300
|
-
float32x4_t c_real_vec = c_vec.val[0];
|
|
301
|
-
float32x4_t c_imag_vec = c_vec.val[1];
|
|
302
|
-
|
|
303
|
-
// Compute the dot product:
|
|
304
|
-
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
305
|
-
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
306
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
307
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
308
|
-
}
|
|
309
|
-
cb_j.real = vaddvq_f32(cb_j_real_vec);
|
|
310
|
-
cb_j.imag = vaddvq_f32(cb_j_imag_vec);
|
|
311
|
-
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
312
|
-
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
// Handle the tail of every row
|
|
316
|
-
simsimd_size_t const tail_length = n % 4;
|
|
317
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
318
|
-
if (tail_length) {
|
|
319
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
320
|
-
simsimd_f32c_t a_i = a[i];
|
|
321
|
-
simsimd_f32c_t cb_j = {0, 0};
|
|
322
|
-
for (simsimd_size_t j = tail_start; j != n; ++j) {
|
|
323
|
-
simsimd_f32c_t b_j = b[j];
|
|
324
|
-
simsimd_f32c_t c_ij = c[i * n + j];
|
|
325
|
-
cb_j.real += b_j.real * c_ij.real - b_j.imag * c_ij.imag;
|
|
326
|
-
cb_j.imag += b_j.real * c_ij.imag + b_j.imag * c_ij.real;
|
|
327
|
-
}
|
|
328
|
-
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
329
|
-
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
results[0] = sum_real;
|
|
334
|
-
results[1] = sum_imag;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
#pragma clang attribute pop
|
|
338
|
-
#pragma GCC pop_options
|
|
339
|
-
#endif // SIMSIMD_TARGET_NEON
|
|
340
|
-
|
|
341
|
-
#if SIMSIMD_TARGET_NEON_F16
|
|
342
|
-
#pragma GCC push_options
|
|
343
|
-
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
344
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
345
|
-
|
|
346
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
|
|
347
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
348
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
349
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
350
|
-
// MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
|
|
351
|
-
float32x4_t a_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
|
|
352
|
-
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
353
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
354
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
|
|
355
|
-
float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
|
|
356
|
-
cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
|
|
357
|
-
}
|
|
358
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
// Handle the tail of every row
|
|
362
|
-
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
363
|
-
simsimd_size_t const tail_length = n % 4;
|
|
364
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
365
|
-
if (tail_length) {
|
|
366
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
367
|
-
simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
|
|
368
|
-
float32x4_t b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
|
|
369
|
-
float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
|
|
370
|
-
simsimd_f32_t cb_j = vaddvq_f32(vmulq_f32(b_vec, c_vec));
|
|
371
|
-
sum += a_i * cb_j;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
*result = sum;
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
|
|
379
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
380
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
381
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
382
|
-
// MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
|
|
383
|
-
float32x4_t a_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
|
|
384
|
-
float32x4_t b_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(b + i))));
|
|
385
|
-
float32x4_t diff_i_vec = vsubq_f32(a_i_vec, b_i_vec);
|
|
386
|
-
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
387
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
388
|
-
float32x4_t a_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(a + j)));
|
|
389
|
-
float32x4_t b_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
|
|
390
|
-
float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
|
|
391
|
-
float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
|
|
392
|
-
cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
393
|
-
}
|
|
394
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// Handle the tail of every row
|
|
398
|
-
simsimd_f32_t sum = vaddvq_f32(sum_vec);
|
|
399
|
-
simsimd_size_t const tail_length = n % 4;
|
|
400
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
401
|
-
if (tail_length) {
|
|
402
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
403
|
-
simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
|
|
404
|
-
simsimd_f32_t b_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + i, 1)));
|
|
405
|
-
simsimd_f32_t diff_i = a_i - b_i;
|
|
406
|
-
float32x4_t a_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + tail_start, tail_length));
|
|
407
|
-
float32x4_t b_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
|
|
408
|
-
float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
|
|
409
|
-
float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
|
|
410
|
-
simsimd_f32_t cdiff_j = vaddvq_f32(vmulq_f32(diff_j_vec, c_vec));
|
|
411
|
-
sum += diff_i * cdiff_j;
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
*result = _simsimd_sqrt_f32_neon(sum);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_reduce_f16x8_neon(float16x8_t vec) {
|
|
419
|
-
// Split the 8-element vector into two 4-element vectors
|
|
420
|
-
float16x4_t low = vget_low_f16(vec); // Lower 4 elements
|
|
421
|
-
float16x4_t high = vget_high_f16(vec); // Upper 4 elements
|
|
422
|
-
|
|
423
|
-
// Add the lower and upper parts
|
|
424
|
-
float16x4_t sum = vadd_f16(low, high);
|
|
425
|
-
|
|
426
|
-
// Perform pairwise addition to reduce 4 elements to 2, then to 1
|
|
427
|
-
sum = vpadd_f16(sum, sum); // First reduction: 4 -> 2
|
|
428
|
-
sum = vpadd_f16(sum, sum); // Second reduction: 2 -> 1
|
|
429
|
-
|
|
430
|
-
// Convert the remaining half-precision value to single-precision and return
|
|
431
|
-
return vgetq_lane_f32(vcvt_f32_f16(sum), 0);
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
SIMSIMD_INTERNAL float16x8x2_t _simsimd_partial_load_f16x8x2_neon(simsimd_f16c_t const *x, simsimd_size_t n) {
|
|
435
|
-
union {
|
|
436
|
-
float16x8x2_t vecs;
|
|
437
|
-
simsimd_f16_t scalars[2][8];
|
|
438
|
-
} result;
|
|
439
|
-
simsimd_size_t i = 0;
|
|
440
|
-
for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
|
|
441
|
-
for (; i < 8; ++i) result.scalars[0][i] = 0, result.scalars[1][i] = 0;
|
|
442
|
-
return result.vecs;
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
|
|
446
|
-
simsimd_f16c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
|
|
447
|
-
simsimd_f32_t sum_real = 0;
|
|
448
|
-
simsimd_f32_t sum_imag = 0;
|
|
449
|
-
simsimd_size_t const tail_length = n % 8;
|
|
450
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
451
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
452
|
-
simsimd_f32c_t a_i = {simsimd_f16_to_f32(&a[i].real), simsimd_f16_to_f32(&a[i].imag)};
|
|
453
|
-
float16x8_t cb_j_real_vec = vdupq_n_f16(0);
|
|
454
|
-
float16x8_t cb_j_imag_vec = vdupq_n_f16(0);
|
|
455
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
456
|
-
// Unpack the input arrays into real and imaginary parts:
|
|
457
|
-
float16x8x2_t b_vec = vld2q_f16((float16_t const *)(b + j));
|
|
458
|
-
float16x8x2_t c_vec = vld2q_f16((float16_t const *)(c + i * n + j));
|
|
459
|
-
float16x8_t b_real_vec = b_vec.val[0];
|
|
460
|
-
float16x8_t b_imag_vec = b_vec.val[1];
|
|
461
|
-
float16x8_t c_real_vec = c_vec.val[0];
|
|
462
|
-
float16x8_t c_imag_vec = c_vec.val[1];
|
|
463
|
-
|
|
464
|
-
// Compute the dot product:
|
|
465
|
-
cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
466
|
-
cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
467
|
-
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
468
|
-
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
469
|
-
}
|
|
470
|
-
// Handle row tails
|
|
471
|
-
if (tail_length) {
|
|
472
|
-
// Unpack the input arrays into real and imaginary parts:
|
|
473
|
-
float16x8x2_t b_vec = _simsimd_partial_load_f16x8x2_neon(b + tail_start, tail_length);
|
|
474
|
-
float16x8x2_t c_vec = _simsimd_partial_load_f16x8x2_neon(c + i * n + tail_start, tail_length);
|
|
475
|
-
float16x8_t b_real_vec = b_vec.val[0];
|
|
476
|
-
float16x8_t b_imag_vec = b_vec.val[1];
|
|
477
|
-
float16x8_t c_real_vec = c_vec.val[0];
|
|
478
|
-
float16x8_t c_imag_vec = c_vec.val[1];
|
|
479
|
-
|
|
480
|
-
// Compute the dot product:
|
|
481
|
-
cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
482
|
-
cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
483
|
-
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
484
|
-
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
simsimd_f32c_t cb_j;
|
|
488
|
-
cb_j.real = _simsimd_reduce_f16x8_neon(cb_j_real_vec);
|
|
489
|
-
cb_j.imag = _simsimd_reduce_f16x8_neon(cb_j_imag_vec);
|
|
490
|
-
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
491
|
-
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
results[0] = sum_real;
|
|
495
|
-
results[1] = sum_imag;
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
#pragma clang attribute pop
|
|
499
|
-
#pragma GCC pop_options
|
|
500
|
-
#endif // SIMSIMD_TARGET_NEON_F16
|
|
501
|
-
|
|
502
|
-
#if SIMSIMD_TARGET_NEON_BF16
|
|
503
|
-
#pragma GCC push_options
|
|
504
|
-
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
505
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
506
|
-
|
|
507
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
508
|
-
simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
|
|
509
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
510
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
511
|
-
float32x4_t a_vec = vdupq_n_f32(simsimd_bf16_to_f32(a + i));
|
|
512
|
-
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
513
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
514
|
-
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
|
|
515
|
-
bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
|
|
516
|
-
cb_j_vec = vbfdotq_f32(cb_j_vec, b_vec, c_vec);
|
|
517
|
-
}
|
|
518
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
// Handle the tail of every row
|
|
522
|
-
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
523
|
-
simsimd_size_t const tail_length = n % 8;
|
|
524
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
525
|
-
if (tail_length) {
|
|
526
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
527
|
-
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
528
|
-
bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
|
|
529
|
-
bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
|
|
530
|
-
simsimd_f32_t cb_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), b_vec, c_vec));
|
|
531
|
-
sum += a_i * cb_j;
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
*result = sum;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
539
|
-
simsimd_bf16_t const *c, simsimd_size_t n,
|
|
540
|
-
simsimd_distance_t *result) {
|
|
541
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
542
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
543
|
-
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
544
|
-
simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
|
|
545
|
-
float32x4_t diff_i_vec = vdupq_n_f32(a_i - b_i);
|
|
546
|
-
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
547
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
548
|
-
bfloat16x8_t a_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(a + j));
|
|
549
|
-
bfloat16x8_t b_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
|
|
550
|
-
|
|
551
|
-
// Arm NEON does not have a native subtraction instruction for `bf16`,
|
|
552
|
-
// so we need to convert to `f32` first, subtract, and only then get back to `bf16`
|
|
553
|
-
// for multiplication.
|
|
554
|
-
float32x4_t a_j_vec_high = vcvt_f32_bf16(vget_high_bf16(a_j_vec));
|
|
555
|
-
float32x4_t a_j_vec_low = vcvt_f32_bf16(vget_low_bf16(a_j_vec));
|
|
556
|
-
float32x4_t b_j_vec_high = vcvt_f32_bf16(vget_high_bf16(b_j_vec));
|
|
557
|
-
float32x4_t b_j_vec_low = vcvt_f32_bf16(vget_low_bf16(b_j_vec));
|
|
558
|
-
float32x4_t diff_j_vec_high = vsubq_f32(a_j_vec_high, b_j_vec_high);
|
|
559
|
-
float32x4_t diff_j_vec_low = vsubq_f32(a_j_vec_low, b_j_vec_low);
|
|
560
|
-
bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
|
|
561
|
-
|
|
562
|
-
bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
|
|
563
|
-
cdiff_j_vec = vbfdotq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
564
|
-
}
|
|
565
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
// Handle the tail of every row
|
|
569
|
-
simsimd_f32_t sum = vaddvq_f32(sum_vec);
|
|
570
|
-
simsimd_size_t const tail_length = n % 8;
|
|
571
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
572
|
-
if (tail_length) {
|
|
573
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
574
|
-
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
575
|
-
simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
|
|
576
|
-
simsimd_f32_t diff_i = a_i - b_i;
|
|
577
|
-
bfloat16x8_t a_j_vec = _simsimd_partial_load_bf16x8_neon(a + tail_start, tail_length);
|
|
578
|
-
bfloat16x8_t b_j_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
|
|
579
|
-
|
|
580
|
-
// Again, upcast for subtraction
|
|
581
|
-
float32x4_t a_j_vec_high = vcvt_f32_bf16(vget_high_bf16(a_j_vec));
|
|
582
|
-
float32x4_t a_j_vec_low = vcvt_f32_bf16(vget_low_bf16(a_j_vec));
|
|
583
|
-
float32x4_t b_j_vec_high = vcvt_f32_bf16(vget_high_bf16(b_j_vec));
|
|
584
|
-
float32x4_t b_j_vec_low = vcvt_f32_bf16(vget_low_bf16(b_j_vec));
|
|
585
|
-
float32x4_t diff_j_vec_high = vsubq_f32(a_j_vec_high, b_j_vec_high);
|
|
586
|
-
float32x4_t diff_j_vec_low = vsubq_f32(a_j_vec_low, b_j_vec_low);
|
|
587
|
-
bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
|
|
588
|
-
|
|
589
|
-
bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
|
|
590
|
-
simsimd_f32_t cdiff_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), diff_j_vec, c_vec));
|
|
591
|
-
sum += diff_i * cdiff_j;
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
*result = _simsimd_sqrt_f32_neon(sum);
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
SIMSIMD_INTERNAL int16x4x2_t _simsimd_partial_load_bf16x4x2_neon(simsimd_bf16c_t const *x, simsimd_size_t n) {
|
|
599
|
-
union {
|
|
600
|
-
int16x4x2_t vec;
|
|
601
|
-
simsimd_bf16_t scalars[2][4];
|
|
602
|
-
} result;
|
|
603
|
-
simsimd_size_t i = 0;
|
|
604
|
-
for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
|
|
605
|
-
for (; i < 4; ++i) result.scalars[1][i] = 0, result.scalars[1][i] = 0;
|
|
606
|
-
return result.vec;
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
|
|
610
|
-
simsimd_bf16c_t const *c, simsimd_size_t n,
|
|
611
|
-
simsimd_distance_t *results) {
|
|
612
|
-
simsimd_f32_t sum_real = 0;
|
|
613
|
-
simsimd_f32_t sum_imag = 0;
|
|
614
|
-
simsimd_size_t const tail_length = n % 4;
|
|
615
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
616
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
617
|
-
simsimd_f32c_t a_i = {simsimd_bf16_to_f32(&a[i].real), simsimd_bf16_to_f32(&a[i].imag)};
|
|
618
|
-
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
619
|
-
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
620
|
-
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
621
|
-
float32x4_t cb_j_real_vec = vdupq_n_f32(0);
|
|
622
|
-
float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
|
|
623
|
-
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
624
|
-
// Unpack the input arrays into real and imaginary parts.
|
|
625
|
-
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
626
|
-
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
627
|
-
int16x4x2_t b_vec = vld2_s16((short const *)(b + j));
|
|
628
|
-
int16x4x2_t c_vec = vld2_s16((short const *)(c + i * n + j));
|
|
629
|
-
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
630
|
-
float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
|
|
631
|
-
float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
|
|
632
|
-
float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
|
|
633
|
-
|
|
634
|
-
// Compute the dot product:
|
|
635
|
-
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
636
|
-
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
637
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
638
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
639
|
-
}
|
|
640
|
-
// Handle row tails
|
|
641
|
-
if (tail_length) {
|
|
642
|
-
// Unpack the input arrays into real and imaginary parts:
|
|
643
|
-
int16x4x2_t b_vec = _simsimd_partial_load_bf16x4x2_neon(b + tail_start, tail_length);
|
|
644
|
-
int16x4x2_t c_vec = _simsimd_partial_load_bf16x4x2_neon(c + i * n + tail_start, tail_length);
|
|
645
|
-
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
646
|
-
float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
|
|
647
|
-
float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
|
|
648
|
-
float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
|
|
649
|
-
|
|
650
|
-
// Compute the dot product:
|
|
651
|
-
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
652
|
-
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
653
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
654
|
-
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
simsimd_f32c_t cb_j;
|
|
658
|
-
cb_j.real = vaddvq_f32(cb_j_real_vec);
|
|
659
|
-
cb_j.imag = vaddvq_f32(cb_j_imag_vec);
|
|
660
|
-
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
661
|
-
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
results[0] = sum_real;
|
|
665
|
-
results[1] = sum_imag;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
#pragma clang attribute pop
|
|
669
|
-
#pragma GCC pop_options
|
|
670
|
-
#endif // SIMSIMD_TARGET_NEON_BF16
|
|
671
|
-
|
|
672
|
-
#endif // _SIMSIMD_TARGET_ARM
|
|
673
|
-
|
|
674
|
-
#if _SIMSIMD_TARGET_X86
|
|
675
|
-
#if SIMSIMD_TARGET_HASWELL
|
|
676
|
-
#pragma GCC push_options
|
|
677
|
-
#pragma GCC target("avx2", "f16c", "fma")
|
|
678
|
-
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
679
|
-
|
|
680
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
|
|
681
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
682
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
683
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
684
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i)));
|
|
685
|
-
__m256 cb_j_vec = _mm256_setzero_ps();
|
|
686
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
687
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j)));
|
|
688
|
-
__m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
689
|
-
cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
690
|
-
}
|
|
691
|
-
sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
// Handle the tail of every row
|
|
695
|
-
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
696
|
-
simsimd_size_t const tail_length = n % 8;
|
|
697
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
698
|
-
if (tail_length) {
|
|
699
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
700
|
-
simsimd_f32_t a_i = _mm256_cvtss_f32(_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))));
|
|
701
|
-
__m256 b_vec = _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length);
|
|
702
|
-
__m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
|
|
703
|
-
simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
|
|
704
|
-
sum += a_i * cb_j;
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
*result = sum;
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
712
|
-
simsimd_f16_t const *c, simsimd_size_t n,
|
|
713
|
-
simsimd_distance_t *result) {
|
|
714
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
715
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
716
|
-
__m256 diff_i_vec = _mm256_sub_ps( //
|
|
717
|
-
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
|
|
718
|
-
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i))));
|
|
719
|
-
__m256 cdiff_j_vec = _mm256_setzero_ps();
|
|
720
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
721
|
-
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
722
|
-
_mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(a + j))),
|
|
723
|
-
_mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j))));
|
|
724
|
-
__m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
725
|
-
cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
726
|
-
}
|
|
727
|
-
sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
// Handle the tail of every row
|
|
731
|
-
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
732
|
-
simsimd_size_t const tail_length = n % 8;
|
|
733
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
734
|
-
if (tail_length) {
|
|
735
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
736
|
-
simsimd_f32_t diff_i = _mm256_cvtss_f32(_mm256_sub_ps( //
|
|
737
|
-
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
|
|
738
|
-
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i)))));
|
|
739
|
-
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
740
|
-
_simsimd_partial_load_f16x8_haswell(a + tail_start, tail_length),
|
|
741
|
-
_simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length));
|
|
742
|
-
__m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
|
|
743
|
-
simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
|
|
744
|
-
sum += diff_i * cdiff_j;
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
*result = _simsimd_sqrt_f32_haswell(sum);
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
752
|
-
simsimd_bf16_t const *c, simsimd_size_t n,
|
|
753
|
-
simsimd_distance_t *result) {
|
|
754
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
755
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
756
|
-
// The `simsimd_bf16_to_f32` is cheaper than `_simsimd_bf16x8_to_f32x8_haswell`
|
|
757
|
-
__m256 a_vec = _mm256_set1_ps(simsimd_bf16_to_f32(a + i));
|
|
758
|
-
__m256 cb_j_vec = _mm256_setzero_ps();
|
|
759
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
760
|
-
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j)));
|
|
761
|
-
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
762
|
-
cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
763
|
-
}
|
|
764
|
-
sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
// Handle the tail of every row
|
|
768
|
-
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
769
|
-
simsimd_size_t const tail_length = n % 8;
|
|
770
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
771
|
-
if (tail_length) {
|
|
772
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
773
|
-
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
774
|
-
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell( //
|
|
775
|
-
_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length));
|
|
776
|
-
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell( //
|
|
777
|
-
_simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
|
|
778
|
-
simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
|
|
779
|
-
sum += a_i * cb_j;
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
*result = sum;
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
787
|
-
simsimd_bf16_t const *c, simsimd_size_t n,
|
|
788
|
-
simsimd_distance_t *result) {
|
|
789
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
790
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
791
|
-
__m256 diff_i_vec = _mm256_sub_ps( //
|
|
792
|
-
_mm256_set1_ps(simsimd_bf16_to_f32(a + i)), //
|
|
793
|
-
_mm256_set1_ps(simsimd_bf16_to_f32(b + i)));
|
|
794
|
-
__m256 cdiff_j_vec = _mm256_setzero_ps();
|
|
795
|
-
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
796
|
-
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
797
|
-
_simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(a + j))), //
|
|
798
|
-
_simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j))));
|
|
799
|
-
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
800
|
-
cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
801
|
-
}
|
|
802
|
-
sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
// Handle the tail of every row
|
|
806
|
-
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
807
|
-
simsimd_size_t const tail_length = n % 8;
|
|
808
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
809
|
-
if (tail_length) {
|
|
810
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
811
|
-
simsimd_f32_t diff_i = simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i);
|
|
812
|
-
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
813
|
-
_simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a + tail_start, tail_length)),
|
|
814
|
-
_simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length)));
|
|
815
|
-
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(
|
|
816
|
-
_simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
|
|
817
|
-
simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
|
|
818
|
-
sum += diff_i * cdiff_j;
|
|
819
|
-
}
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
*result = _simsimd_sqrt_f32_haswell(sum);
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
#pragma clang attribute pop
|
|
826
|
-
#pragma GCC pop_options
|
|
827
|
-
#endif // SIMSIMD_TARGET_HASWELL
|
|
828
|
-
|
|
829
|
-
#if SIMSIMD_TARGET_SKYLAKE
|
|
830
|
-
#pragma GCC push_options
|
|
831
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
|
|
832
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
833
|
-
|
|
834
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake_under16unrolled(simsimd_f32_t const *a, simsimd_f32_t const *b,
|
|
835
|
-
simsimd_f32_t const *c, simsimd_size_t n,
|
|
836
|
-
simsimd_distance_t *result) {
|
|
837
|
-
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
838
|
-
// until the very end of the computation.
|
|
839
|
-
__mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
840
|
-
__m512 const b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
841
|
-
|
|
842
|
-
__m512 cb_j1 = _mm512_setzero_ps();
|
|
843
|
-
__m512 cb_j2 = _mm512_setzero_ps();
|
|
844
|
-
__m512 cb_j3 = _mm512_setzero_ps();
|
|
845
|
-
__m512 cb_j4 = _mm512_setzero_ps();
|
|
846
|
-
|
|
847
|
-
// Unroll the loop to process 4x ZMM registers at a time.
|
|
848
|
-
simsimd_size_t i = 0;
|
|
849
|
-
for (; i + 4 <= n; i += 4) {
|
|
850
|
-
cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
|
|
851
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
|
|
852
|
-
cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
|
|
853
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
|
|
854
|
-
cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
|
|
855
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
|
|
856
|
-
cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
|
|
857
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
if (i + 0 < n)
|
|
861
|
-
cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
|
|
862
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
|
|
863
|
-
if (i + 1 < n)
|
|
864
|
-
cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
|
|
865
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
|
|
866
|
-
if (i + 2 < n)
|
|
867
|
-
cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
|
|
868
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
|
|
869
|
-
if (i + 3 < n)
|
|
870
|
-
cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
|
|
871
|
-
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
|
|
872
|
-
|
|
873
|
-
// Combine cb_j sums
|
|
874
|
-
__m512 sum_vec = _mm512_add_ps( //
|
|
875
|
-
_mm512_add_ps(cb_j1, cb_j2), //
|
|
876
|
-
_mm512_add_ps(cb_j3, cb_j4));
|
|
877
|
-
*result = _mm512_reduce_add_ps(sum_vec);
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
|
|
881
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
882
|
-
|
|
883
|
-
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
|
|
884
|
-
// per row and 16 rows at a time, still keeping enough register space for temporaries.
|
|
885
|
-
if (n <= 16) {
|
|
886
|
-
simsimd_bilinear_f32_skylake_under16unrolled(a, b, c, n, result);
|
|
887
|
-
return;
|
|
888
|
-
}
|
|
889
|
-
|
|
890
|
-
// Default case for arbitrary size `n`
|
|
891
|
-
simsimd_size_t const tail_length = n % 16;
|
|
892
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
893
|
-
__m512 sum_vec = _mm512_setzero_ps();
|
|
894
|
-
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
895
|
-
|
|
896
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
897
|
-
__m512 a_vec = _mm512_set1_ps(a[i]);
|
|
898
|
-
__m512 cb_j_vec = _mm512_setzero_ps();
|
|
899
|
-
__m512 b_vec, c_vec;
|
|
900
|
-
simsimd_size_t j = 0;
|
|
901
|
-
|
|
902
|
-
simsimd_bilinear_f32_skylake_cycle:
|
|
903
|
-
if (j + 16 <= n) {
|
|
904
|
-
b_vec = _mm512_loadu_ps(b + j);
|
|
905
|
-
c_vec = _mm512_loadu_ps(c + i * n + j);
|
|
906
|
-
}
|
|
907
|
-
else {
|
|
908
|
-
b_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
|
|
909
|
-
c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
|
|
910
|
-
}
|
|
911
|
-
cb_j_vec = _mm512_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
912
|
-
j += 16;
|
|
913
|
-
if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
|
|
914
|
-
sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
*result = _mm512_reduce_add_ps(sum_vec);
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b,
|
|
921
|
-
simsimd_f32_t const *c, simsimd_size_t n,
|
|
922
|
-
simsimd_distance_t *result) {
|
|
923
|
-
simsimd_size_t const tail_length = n % 16;
|
|
924
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
925
|
-
__m512 sum_vec = _mm512_setzero_ps();
|
|
926
|
-
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
927
|
-
|
|
928
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
929
|
-
__m512 diff_i_vec = _mm512_set1_ps(a[i] - b[i]);
|
|
930
|
-
__m512 cdiff_j_vec = _mm512_setzero_ps(), cdiff_j_bot_vec = _mm512_setzero_ps();
|
|
931
|
-
__m512 a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
932
|
-
simsimd_size_t j = 0;
|
|
933
|
-
|
|
934
|
-
// The nested loop is cleaner to implement with a `goto` in this case:
|
|
935
|
-
simsimd_bilinear_f32_skylake_cycle:
|
|
936
|
-
if (j + 16 <= n) {
|
|
937
|
-
a_j_vec = _mm512_loadu_ps(a + j);
|
|
938
|
-
b_j_vec = _mm512_loadu_ps(b + j);
|
|
939
|
-
c_vec = _mm512_loadu_ps(c + i * n + j);
|
|
940
|
-
}
|
|
941
|
-
else {
|
|
942
|
-
a_j_vec = _mm512_maskz_loadu_ps(tail_mask, a + tail_start);
|
|
943
|
-
b_j_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
|
|
944
|
-
c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
|
|
945
|
-
}
|
|
946
|
-
diff_j_vec = _mm512_sub_ps(a_j_vec, b_j_vec);
|
|
947
|
-
cdiff_j_vec = _mm512_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
948
|
-
j += 16;
|
|
949
|
-
if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
|
|
950
|
-
sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
*result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_ps(sum_vec));
|
|
954
|
-
}
|
|
955
|
-
|
|
956
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
|
|
957
|
-
simsimd_f32c_t const *c, simsimd_size_t n,
|
|
958
|
-
simsimd_distance_t *results) {
|
|
959
|
-
|
|
960
|
-
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
961
|
-
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
962
|
-
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
963
|
-
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
964
|
-
// one of the vectors.
|
|
965
|
-
__m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
|
|
966
|
-
|
|
967
|
-
// Default case for arbitrary size `n`
|
|
968
|
-
simsimd_size_t const tail_length = n % 8;
|
|
969
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
970
|
-
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
971
|
-
simsimd_f32_t sum_real = 0;
|
|
972
|
-
simsimd_f32_t sum_imag = 0;
|
|
973
|
-
|
|
974
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
975
|
-
simsimd_f32_t const a_i_real = a[i].real;
|
|
976
|
-
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
977
|
-
__m512 cb_j_real_vec = _mm512_setzero_ps();
|
|
978
|
-
__m512 cb_j_imag_vec = _mm512_setzero_ps();
|
|
979
|
-
__m512 b_vec, c_vec;
|
|
980
|
-
simsimd_size_t j = 0;
|
|
981
|
-
|
|
982
|
-
simsimd_bilinear_f32c_skylake_cycle:
|
|
983
|
-
if (j + 8 <= n) {
|
|
984
|
-
b_vec = _mm512_loadu_ps((simsimd_f32_t const *)(b + j));
|
|
985
|
-
c_vec = _mm512_loadu_ps((simsimd_f32_t const *)(c + i * n + j));
|
|
986
|
-
}
|
|
987
|
-
else {
|
|
988
|
-
b_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(b + tail_start));
|
|
989
|
-
c_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(c + i * n + tail_start));
|
|
990
|
-
}
|
|
991
|
-
// The real part of the product: b.real * c.real - b.imag * c.imag.
|
|
992
|
-
// The subtraction will be performed later with a sign flip.
|
|
993
|
-
cb_j_real_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_real_vec);
|
|
994
|
-
// The imaginary part of the product: b.real * c.imag + b.imag * c.real.
|
|
995
|
-
// Swap the imaginary and real parts of `c` before multiplication:
|
|
996
|
-
c_vec = _mm512_permute_ps(c_vec, 0xB1); //? Swap adjacent entries within each pair
|
|
997
|
-
cb_j_imag_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_imag_vec);
|
|
998
|
-
j += 8;
|
|
999
|
-
if (j < n) goto simsimd_bilinear_f32c_skylake_cycle;
|
|
1000
|
-
// Flip the sign bit in every second scalar before accumulation:
|
|
1001
|
-
cb_j_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(cb_j_real_vec), sign_flip_vec));
|
|
1002
|
-
// Horizontal sums are the expensive part of the computation:
|
|
1003
|
-
simsimd_f32_t const cb_j_real = _mm512_reduce_add_ps(cb_j_real_vec);
|
|
1004
|
-
simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ps(cb_j_imag_vec);
|
|
1005
|
-
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1006
|
-
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
// Reduce horizontal sums:
|
|
1010
|
-
results[0] = sum_real;
|
|
1011
|
-
results[1] = sum_imag;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake_under8unrolled(simsimd_f64_t const *a, simsimd_f64_t const *b,
|
|
1015
|
-
simsimd_f64_t const *c, simsimd_size_t n,
|
|
1016
|
-
simsimd_distance_t *result) {
|
|
1017
|
-
|
|
1018
|
-
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
1019
|
-
// until the very end of the computation.
|
|
1020
|
-
__mmask8 const row_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1021
|
-
__m512d const b_vec = _mm512_maskz_loadu_pd(row_mask, b);
|
|
1022
|
-
|
|
1023
|
-
__m512d cb_j1 = _mm512_setzero_pd();
|
|
1024
|
-
__m512d cb_j2 = _mm512_setzero_pd();
|
|
1025
|
-
__m512d cb_j3 = _mm512_setzero_pd();
|
|
1026
|
-
__m512d cb_j4 = _mm512_setzero_pd();
|
|
1027
|
-
|
|
1028
|
-
// clang-format off
|
|
1029
|
-
if (n > 0) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 0), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[0])), cb_j1);
|
|
1030
|
-
if (n > 1) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 1), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[1])), cb_j2);
|
|
1031
|
-
if (n > 2) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 2), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[2])), cb_j3);
|
|
1032
|
-
if (n > 3) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 3), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[3])), cb_j4);
|
|
1033
|
-
|
|
1034
|
-
if (n > 4) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 4), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[4])), cb_j1);
|
|
1035
|
-
if (n > 5) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 5), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[5])), cb_j2);
|
|
1036
|
-
if (n > 6) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 6), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[6])), cb_j3);
|
|
1037
|
-
if (n > 7) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 7), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[7])), cb_j4);
|
|
1038
|
-
// clang-format on
|
|
1039
|
-
|
|
1040
|
-
// Combine cb_j sums
|
|
1041
|
-
__m512d sum_vec = _mm512_add_pd( //
|
|
1042
|
-
_mm512_add_pd(cb_j1, cb_j2), //
|
|
1043
|
-
_mm512_add_pd(cb_j3, cb_j4));
|
|
1044
|
-
*result = _mm512_reduce_add_pd(sum_vec);
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c,
|
|
1048
|
-
simsimd_size_t n, simsimd_distance_t *result) {
|
|
1049
|
-
|
|
1050
|
-
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
|
|
1051
|
-
// per row and 8 rows at a time, still keeping enough register space for temporaries.
|
|
1052
|
-
if (n <= 8) {
|
|
1053
|
-
simsimd_bilinear_f64_skylake_under8unrolled(a, b, c, n, result);
|
|
1054
|
-
return;
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
// Default case for arbitrary size `n`
|
|
1058
|
-
simsimd_size_t const tail_length = n % 8;
|
|
1059
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1060
|
-
__m512d sum_vec = _mm512_setzero_pd();
|
|
1061
|
-
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1062
|
-
|
|
1063
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1064
|
-
__m512d a_vec = _mm512_set1_pd(a[i]);
|
|
1065
|
-
__m512d cb_j_vec = _mm512_setzero_pd();
|
|
1066
|
-
__m512d b_vec, c_vec;
|
|
1067
|
-
simsimd_size_t j = 0;
|
|
1068
|
-
|
|
1069
|
-
simsimd_bilinear_f64_skylake_cycle:
|
|
1070
|
-
if (j + 8 <= n) {
|
|
1071
|
-
b_vec = _mm512_loadu_pd(b + j);
|
|
1072
|
-
c_vec = _mm512_loadu_pd(c + i * n + j);
|
|
1073
|
-
}
|
|
1074
|
-
else {
|
|
1075
|
-
b_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
|
|
1076
|
-
c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
|
|
1077
|
-
}
|
|
1078
|
-
cb_j_vec = _mm512_fmadd_pd(b_vec, c_vec, cb_j_vec);
|
|
1079
|
-
j += 8;
|
|
1080
|
-
if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
|
|
1081
|
-
sum_vec = _mm512_fmadd_pd(a_vec, cb_j_vec, sum_vec);
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
*result = _mm512_reduce_add_pd(sum_vec);
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b,
|
|
1088
|
-
simsimd_f64_t const *c, simsimd_size_t n,
|
|
1089
|
-
simsimd_distance_t *result) {
|
|
1090
|
-
simsimd_size_t const tail_length = n % 8;
|
|
1091
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1092
|
-
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1093
|
-
__m512d sum_vec = _mm512_setzero_pd();
|
|
1094
|
-
|
|
1095
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1096
|
-
__m512d diff_i_vec = _mm512_set1_pd(a[i] - b[i]);
|
|
1097
|
-
__m512d cdiff_j_vec = _mm512_setzero_pd();
|
|
1098
|
-
__m512d a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
1099
|
-
simsimd_size_t j = 0;
|
|
1100
|
-
|
|
1101
|
-
// The nested loop is cleaner to implement with a `goto` in this case:
|
|
1102
|
-
simsimd_bilinear_f64_skylake_cycle:
|
|
1103
|
-
if (j + 8 <= n) {
|
|
1104
|
-
a_j_vec = _mm512_loadu_pd(a + j);
|
|
1105
|
-
b_j_vec = _mm512_loadu_pd(b + j);
|
|
1106
|
-
c_vec = _mm512_loadu_pd(c + i * n + j);
|
|
1107
|
-
}
|
|
1108
|
-
else {
|
|
1109
|
-
a_j_vec = _mm512_maskz_loadu_pd(tail_mask, a + tail_start);
|
|
1110
|
-
b_j_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
|
|
1111
|
-
c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
|
|
1112
|
-
}
|
|
1113
|
-
diff_j_vec = _mm512_sub_pd(a_j_vec, b_j_vec);
|
|
1114
|
-
cdiff_j_vec = _mm512_fmadd_pd(diff_j_vec, c_vec, cdiff_j_vec);
|
|
1115
|
-
j += 8;
|
|
1116
|
-
if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
|
|
1117
|
-
sum_vec = _mm512_fmadd_pd(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
*result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_pd(sum_vec));
|
|
1121
|
-
}
|
|
1122
|
-
|
|
1123
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const *a, simsimd_f64c_t const *b,
|
|
1124
|
-
simsimd_f64c_t const *c, simsimd_size_t n,
|
|
1125
|
-
simsimd_distance_t *results) {
|
|
1126
|
-
|
|
1127
|
-
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1128
|
-
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1129
|
-
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1130
|
-
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1131
|
-
// one of the vectors.
|
|
1132
|
-
__m512i const sign_flip_vec = _mm512_set_epi64( //
|
|
1133
|
-
0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
|
|
1134
|
-
0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
|
|
1135
|
-
);
|
|
1136
|
-
|
|
1137
|
-
// Default case for arbitrary size `n`
|
|
1138
|
-
simsimd_size_t const tail_length = n % 4;
|
|
1139
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1140
|
-
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1141
|
-
simsimd_f64_t sum_real = 0;
|
|
1142
|
-
simsimd_f64_t sum_imag = 0;
|
|
1143
|
-
|
|
1144
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1145
|
-
simsimd_f64_t const a_i_real = a[i].real;
|
|
1146
|
-
simsimd_f64_t const a_i_imag = a[i].imag;
|
|
1147
|
-
__m512d cb_j_real_vec = _mm512_setzero_pd();
|
|
1148
|
-
__m512d cb_j_imag_vec = _mm512_setzero_pd();
|
|
1149
|
-
__m512d b_vec, c_vec;
|
|
1150
|
-
simsimd_size_t j = 0;
|
|
1151
|
-
|
|
1152
|
-
simsimd_bilinear_f64c_skylake_cycle:
|
|
1153
|
-
if (j + 4 <= n) {
|
|
1154
|
-
b_vec = _mm512_loadu_pd((simsimd_f64_t const *)(b + j));
|
|
1155
|
-
c_vec = _mm512_loadu_pd((simsimd_f64_t const *)(c + i * n + j));
|
|
1156
|
-
}
|
|
1157
|
-
else {
|
|
1158
|
-
b_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(b + tail_start));
|
|
1159
|
-
c_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(c + i * n + tail_start));
|
|
1160
|
-
}
|
|
1161
|
-
// The real part of the product: b.real * c.real - b.imag * c.imag.
|
|
1162
|
-
// The subtraction will be performed later with a sign flip.
|
|
1163
|
-
cb_j_real_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_real_vec);
|
|
1164
|
-
// The imaginary part of the product: b.real * c.imag + b.imag * c.real.
|
|
1165
|
-
// Swap the imaginary and real parts of `c` before multiplication:
|
|
1166
|
-
c_vec = _mm512_permute_pd(c_vec, 0x55); //? Same as 0b01010101.
|
|
1167
|
-
cb_j_imag_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_imag_vec);
|
|
1168
|
-
j += 4;
|
|
1169
|
-
if (j < n) goto simsimd_bilinear_f64c_skylake_cycle;
|
|
1170
|
-
// Flip the sign bit in every second scalar before accumulation:
|
|
1171
|
-
cb_j_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(cb_j_real_vec), sign_flip_vec));
|
|
1172
|
-
// Horizontal sums are the expensive part of the computation:
|
|
1173
|
-
simsimd_f64_t const cb_j_real = _mm512_reduce_add_pd(cb_j_real_vec);
|
|
1174
|
-
simsimd_f64_t const cb_j_imag = _mm512_reduce_add_pd(cb_j_imag_vec);
|
|
1175
|
-
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1176
|
-
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
// Reduce horizontal sums:
|
|
1180
|
-
results[0] = sum_real;
|
|
1181
|
-
results[1] = sum_imag;
|
|
1182
|
-
}
|
|
1183
|
-
|
|
1184
|
-
#pragma clang attribute pop
|
|
1185
|
-
#pragma GCC pop_options
|
|
1186
|
-
#endif // SIMSIMD_TARGET_SKYLAKE
|
|
1187
|
-
|
|
1188
|
-
#if SIMSIMD_TARGET_GENOA
|
|
1189
|
-
#pragma GCC push_options
|
|
1190
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1191
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1192
|
-
apply_to = function)
|
|
1193
|
-
|
|
1194
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
1195
|
-
simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
|
|
1196
|
-
simsimd_size_t const tail_length = n % 32;
|
|
1197
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1198
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1199
|
-
__m512 sum_vec = _mm512_setzero_ps();
|
|
1200
|
-
|
|
1201
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1202
|
-
__m512 a_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i));
|
|
1203
|
-
__m512 cb_j_vec = _mm512_setzero_ps();
|
|
1204
|
-
__m512i b_vec, c_vec;
|
|
1205
|
-
simsimd_size_t j = 0;
|
|
1206
|
-
|
|
1207
|
-
simsimd_bilinear_bf16_genoa_cycle:
|
|
1208
|
-
if (j + 32 <= n) {
|
|
1209
|
-
b_vec = _mm512_loadu_epi16(b + j);
|
|
1210
|
-
c_vec = _mm512_loadu_epi16(c + i * n + j);
|
|
1211
|
-
}
|
|
1212
|
-
else {
|
|
1213
|
-
b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
1214
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
1215
|
-
}
|
|
1216
|
-
cb_j_vec = _mm512_dpbf16_ps(cb_j_vec, (__m512bh)(b_vec), (__m512bh)(c_vec));
|
|
1217
|
-
j += 32;
|
|
1218
|
-
if (j < n) goto simsimd_bilinear_bf16_genoa_cycle;
|
|
1219
|
-
sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
*result = _mm512_reduce_add_ps(sum_vec);
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
1226
|
-
simsimd_bf16_t const *c, simsimd_size_t n,
|
|
1227
|
-
simsimd_distance_t *result) {
|
|
1228
|
-
simsimd_size_t const tail_length = n % 32;
|
|
1229
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1230
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1231
|
-
__m512 sum_vec = _mm512_setzero_ps();
|
|
1232
|
-
|
|
1233
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1234
|
-
__m512 diff_i_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i));
|
|
1235
|
-
__m512 cdiff_j_vec = _mm512_setzero_ps();
|
|
1236
|
-
__m512i a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
1237
|
-
simsimd_size_t j = 0;
|
|
1238
|
-
|
|
1239
|
-
// The nested loop is cleaner to implement with a `goto` in this case:
|
|
1240
|
-
simsimd_mahalanobis_bf16_genoa_cycle:
|
|
1241
|
-
if (j + 32 <= n) {
|
|
1242
|
-
a_j_vec = _mm512_loadu_epi16(a + j);
|
|
1243
|
-
b_j_vec = _mm512_loadu_epi16(b + j);
|
|
1244
|
-
c_vec = _mm512_loadu_epi16(c + i * n + j);
|
|
1245
|
-
}
|
|
1246
|
-
else {
|
|
1247
|
-
a_j_vec = _mm512_maskz_loadu_epi16(tail_mask, a + tail_start);
|
|
1248
|
-
b_j_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
1249
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
1250
|
-
}
|
|
1251
|
-
diff_j_vec = _simsimd_substract_bf16x32_genoa(a_j_vec, b_j_vec);
|
|
1252
|
-
cdiff_j_vec = _mm512_dpbf16_ps(cdiff_j_vec, (__m512bh)(diff_j_vec), (__m512bh)(c_vec));
|
|
1253
|
-
j += 32;
|
|
1254
|
-
if (j < n) goto simsimd_mahalanobis_bf16_genoa_cycle;
|
|
1255
|
-
sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
1256
|
-
}
|
|
1257
|
-
|
|
1258
|
-
*result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ps(sum_vec));
|
|
1259
|
-
}
|
|
1260
|
-
|
|
1261
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
|
|
1262
|
-
simsimd_bf16c_t const *c, simsimd_size_t n,
|
|
1263
|
-
simsimd_distance_t *results) {
|
|
1264
|
-
|
|
1265
|
-
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1266
|
-
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1267
|
-
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1268
|
-
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1269
|
-
// one of the vectors.
|
|
1270
|
-
__m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
|
|
1271
|
-
__m512i const swap_adjacent_vec = _mm512_set_epi8( //
|
|
1272
|
-
61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
|
|
1273
|
-
45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
|
|
1274
|
-
29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
|
|
1275
|
-
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
|
|
1276
|
-
);
|
|
1277
|
-
|
|
1278
|
-
// Default case for arbitrary size `n`
|
|
1279
|
-
simsimd_size_t const tail_length = n % 16;
|
|
1280
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1281
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1282
|
-
simsimd_f64_t sum_real = 0;
|
|
1283
|
-
simsimd_f64_t sum_imag = 0;
|
|
1284
|
-
|
|
1285
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1286
|
-
simsimd_f32_t const a_i_real = a[i].real;
|
|
1287
|
-
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
1288
|
-
__m512 cb_j_real_vec = _mm512_setzero_ps();
|
|
1289
|
-
__m512 cb_j_imag_vec = _mm512_setzero_ps();
|
|
1290
|
-
__m512i b_vec, c_vec;
|
|
1291
|
-
simsimd_size_t j = 0;
|
|
1292
|
-
|
|
1293
|
-
simsimd_bilinear_bf16c_skylake_cycle:
|
|
1294
|
-
if (j + 16 <= n) {
|
|
1295
|
-
b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
|
|
1296
|
-
c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
|
|
1297
|
-
}
|
|
1298
|
-
else {
|
|
1299
|
-
b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
|
|
1300
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
|
|
1301
|
-
}
|
|
1302
|
-
cb_j_real_vec = _mm512_dpbf16_ps( //
|
|
1303
|
-
cb_j_real_vec, //
|
|
1304
|
-
(__m512bh)(_mm512_xor_si512(c_vec, sign_flip_vec)), //
|
|
1305
|
-
(__m512bh)b_vec);
|
|
1306
|
-
cb_j_imag_vec = _mm512_dpbf16_ps( //
|
|
1307
|
-
cb_j_imag_vec, //
|
|
1308
|
-
(__m512bh)(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
|
|
1309
|
-
(__m512bh)b_vec);
|
|
1310
|
-
j += 16;
|
|
1311
|
-
if (j < n) goto simsimd_bilinear_bf16c_skylake_cycle;
|
|
1312
|
-
// Horizontal sums are the expensive part of the computation:
|
|
1313
|
-
simsimd_f64_t const cb_j_real = _simsimd_reduce_f32x16_skylake(cb_j_real_vec);
|
|
1314
|
-
simsimd_f64_t const cb_j_imag = _simsimd_reduce_f32x16_skylake(cb_j_imag_vec);
|
|
1315
|
-
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1316
|
-
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
// Reduce horizontal sums:
|
|
1320
|
-
results[0] = sum_real;
|
|
1321
|
-
results[1] = sum_imag;
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
#pragma clang attribute pop
|
|
1325
|
-
#pragma GCC pop_options
|
|
1326
|
-
#endif // SIMSIMD_TARGET_GENOA
|
|
1327
|
-
|
|
1328
|
-
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1329
|
-
#pragma GCC push_options
|
|
1330
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1331
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
1332
|
-
apply_to = function)
|
|
1333
|
-
|
|
1334
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire_under32unrolled(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
1335
|
-
simsimd_f16_t const *c, simsimd_size_t const n,
|
|
1336
|
-
simsimd_distance_t *result) {
|
|
1337
|
-
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
1338
|
-
// until the very end of the computation.
|
|
1339
|
-
__mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1340
|
-
__m512h const b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1341
|
-
|
|
1342
|
-
// Independently accumulate the partial sums into separate variables to avoid data-dependencies.
|
|
1343
|
-
__m512h cb_j1 = _mm512_setzero_ph();
|
|
1344
|
-
__m512h cb_j2 = _mm512_setzero_ph();
|
|
1345
|
-
__m512h cb_j3 = _mm512_setzero_ph();
|
|
1346
|
-
__m512h cb_j4 = _mm512_setzero_ph();
|
|
1347
|
-
|
|
1348
|
-
// Unroll the loop to process 4x ZMM registers at a time.
|
|
1349
|
-
simsimd_size_t i = 0;
|
|
1350
|
-
for (; i + 4 <= n; i += 4) {
|
|
1351
|
-
// If the code is compiled without native support for `_Float16`, we need a workaround
|
|
1352
|
-
// to avoid implicit casts from out `simsimd_f16_t` to `_Float16`.
|
|
1353
|
-
cb_j1 = _mm512_fmadd_ph(
|
|
1354
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
|
|
1355
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
|
|
1356
|
-
cb_j2 = _mm512_fmadd_ph(
|
|
1357
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
|
|
1358
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
|
|
1359
|
-
cb_j3 = _mm512_fmadd_ph(
|
|
1360
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
|
|
1361
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
|
|
1362
|
-
cb_j4 = _mm512_fmadd_ph(
|
|
1363
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
|
|
1364
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
// Handle the tail of the loop:
|
|
1368
|
-
if (i + 0 < n)
|
|
1369
|
-
cb_j1 = _mm512_fmadd_ph(
|
|
1370
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
|
|
1371
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
|
|
1372
|
-
if (i + 1 < n)
|
|
1373
|
-
cb_j2 = _mm512_fmadd_ph(
|
|
1374
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
|
|
1375
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
|
|
1376
|
-
if (i + 2 < n)
|
|
1377
|
-
cb_j3 = _mm512_fmadd_ph(
|
|
1378
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
|
|
1379
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
|
|
1380
|
-
if (i + 3 < n)
|
|
1381
|
-
cb_j4 = _mm512_fmadd_ph(
|
|
1382
|
-
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
|
|
1383
|
-
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
|
|
1384
|
-
|
|
1385
|
-
// Combine cb_j sums
|
|
1386
|
-
__m512h sum_vec = _mm512_add_ph( //
|
|
1387
|
-
_mm512_add_ph(cb_j1, cb_j2), //
|
|
1388
|
-
_mm512_add_ph(cb_j3, cb_j4));
|
|
1389
|
-
*result = _mm512_reduce_add_ph(sum_vec);
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
1393
|
-
simsimd_f16_t const *c, simsimd_size_t n,
|
|
1394
|
-
simsimd_distance_t *result) {
|
|
1395
|
-
|
|
1396
|
-
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 32 cells
|
|
1397
|
-
// per row and 32 rows at a time, still keeping enough register space for temporaries.
|
|
1398
|
-
if (n <= 32) {
|
|
1399
|
-
simsimd_bilinear_f16_sapphire_under32unrolled(a, b, c, n, result);
|
|
1400
|
-
return;
|
|
1401
|
-
}
|
|
1402
|
-
|
|
1403
|
-
simsimd_size_t const tail_length = n % 32;
|
|
1404
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1405
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1406
|
-
__m512h sum_vec = _mm512_setzero_ph();
|
|
1407
|
-
|
|
1408
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1409
|
-
__m512h a_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
|
|
1410
|
-
__m512h cb_j_vec = _mm512_setzero_ph();
|
|
1411
|
-
__m512i b_vec, c_vec;
|
|
1412
|
-
simsimd_size_t j = 0;
|
|
1413
|
-
|
|
1414
|
-
simsimd_bilinear_f16_sapphire_cycle:
|
|
1415
|
-
if (j + 32 <= n) {
|
|
1416
|
-
b_vec = _mm512_loadu_epi16(b + j);
|
|
1417
|
-
c_vec = _mm512_loadu_epi16(c + i * n + j);
|
|
1418
|
-
}
|
|
1419
|
-
else {
|
|
1420
|
-
b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
1421
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
1422
|
-
}
|
|
1423
|
-
cb_j_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(c_vec), cb_j_vec);
|
|
1424
|
-
j += 32;
|
|
1425
|
-
if (j < n) goto simsimd_bilinear_f16_sapphire_cycle;
|
|
1426
|
-
sum_vec = _mm512_fmadd_ph(a_vec, cb_j_vec, sum_vec);
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
*result = _mm512_reduce_add_ph(sum_vec);
|
|
1430
|
-
}
|
|
1431
|
-
|
|
1432
|
-
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
1433
|
-
simsimd_f16_t const *c, simsimd_size_t n,
|
|
1434
|
-
simsimd_distance_t *result) {
|
|
1435
|
-
simsimd_size_t const tail_length = n % 32;
|
|
1436
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1437
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1438
|
-
__m512h sum_vec = _mm512_setzero_ph();
|
|
1439
|
-
|
|
1440
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1441
|
-
__m512h a_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
|
|
1442
|
-
__m512h b_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(b + i)));
|
|
1443
|
-
__m512h diff_i_vec = _mm512_sub_ph(a_i_vec, b_i_vec);
|
|
1444
|
-
__m512h cdiff_j_vec = _mm512_setzero_ph();
|
|
1445
|
-
__m512h diff_j_vec;
|
|
1446
|
-
__m512i a_j_vec, b_j_vec, c_vec;
|
|
1447
|
-
simsimd_size_t j = 0;
|
|
1448
|
-
|
|
1449
|
-
// The nested loop is cleaner to implement with a `goto` in this case:
|
|
1450
|
-
simsimd_mahalanobis_f16_sapphire_cycle:
|
|
1451
|
-
if (j + 32 <= n) {
|
|
1452
|
-
a_j_vec = _mm512_loadu_epi16(a + j);
|
|
1453
|
-
b_j_vec = _mm512_loadu_epi16(b + j);
|
|
1454
|
-
c_vec = _mm512_loadu_epi16(c + i * n + j);
|
|
1455
|
-
}
|
|
1456
|
-
else {
|
|
1457
|
-
a_j_vec = _mm512_maskz_loadu_epi16(tail_mask, a + tail_start);
|
|
1458
|
-
b_j_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
1459
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
1460
|
-
}
|
|
1461
|
-
diff_j_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_j_vec), _mm512_castsi512_ph(b_j_vec));
|
|
1462
|
-
cdiff_j_vec = _mm512_fmadd_ph(diff_j_vec, _mm512_castsi512_ph(c_vec), cdiff_j_vec);
|
|
1463
|
-
j += 32;
|
|
1464
|
-
if (j < n) goto simsimd_mahalanobis_f16_sapphire_cycle;
|
|
1465
|
-
sum_vec = _mm512_fmadd_ph(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
*result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ph(sum_vec));
|
|
1469
|
-
}
|
|
1470
|
-
|
|
1471
|
-
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
|
|
1472
|
-
simsimd_f16c_t const *c, simsimd_size_t n,
|
|
1473
|
-
simsimd_distance_t *results) {
|
|
1474
|
-
|
|
1475
|
-
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1476
|
-
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1477
|
-
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1478
|
-
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1479
|
-
// one of the vectors.
|
|
1480
|
-
__m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
|
|
1481
|
-
__m512i const swap_adjacent_vec = _mm512_set_epi8( //
|
|
1482
|
-
61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
|
|
1483
|
-
45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
|
|
1484
|
-
29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
|
|
1485
|
-
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
|
|
1486
|
-
);
|
|
1487
|
-
|
|
1488
|
-
// Default case for arbitrary size `n`
|
|
1489
|
-
simsimd_size_t const tail_length = n % 16;
|
|
1490
|
-
simsimd_size_t const tail_start = n - tail_length;
|
|
1491
|
-
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1492
|
-
simsimd_f32_t sum_real = 0;
|
|
1493
|
-
simsimd_f32_t sum_imag = 0;
|
|
1494
|
-
|
|
1495
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1496
|
-
simsimd_f32_t const a_i_real = a[i].real;
|
|
1497
|
-
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
1498
|
-
__m512h cb_j_real_vec = _mm512_setzero_ph();
|
|
1499
|
-
__m512h cb_j_imag_vec = _mm512_setzero_ph();
|
|
1500
|
-
__m512i b_vec, c_vec;
|
|
1501
|
-
simsimd_size_t j = 0;
|
|
1502
|
-
|
|
1503
|
-
simsimd_bilinear_f16c_skylake_cycle:
|
|
1504
|
-
if (j + 16 <= n) {
|
|
1505
|
-
b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
|
|
1506
|
-
c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
|
|
1507
|
-
}
|
|
1508
|
-
else {
|
|
1509
|
-
b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
|
|
1510
|
-
c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
|
|
1511
|
-
}
|
|
1512
|
-
cb_j_real_vec = _mm512_fmadd_ph( //
|
|
1513
|
-
_mm512_castsi512_ph(_mm512_xor_si512(c_vec, sign_flip_vec)), //
|
|
1514
|
-
_mm512_castsi512_ph(b_vec), cb_j_real_vec);
|
|
1515
|
-
cb_j_imag_vec = _mm512_fmadd_ph( //
|
|
1516
|
-
_mm512_castsi512_ph(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
|
|
1517
|
-
_mm512_castsi512_ph(b_vec), cb_j_imag_vec);
|
|
1518
|
-
j += 16;
|
|
1519
|
-
if (j < n) goto simsimd_bilinear_f16c_skylake_cycle;
|
|
1520
|
-
// Horizontal sums are the expensive part of the computation:
|
|
1521
|
-
simsimd_f32_t const cb_j_real = _mm512_reduce_add_ph(cb_j_real_vec);
|
|
1522
|
-
simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ph(cb_j_imag_vec);
|
|
1523
|
-
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1524
|
-
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1525
|
-
}
|
|
1526
|
-
|
|
1527
|
-
// Reduce horizontal sums:
|
|
1528
|
-
results[0] = sum_real;
|
|
1529
|
-
results[1] = sum_imag;
|
|
1530
|
-
}
|
|
1531
|
-
|
|
1532
|
-
#pragma clang attribute pop
|
|
1533
|
-
#pragma GCC pop_options
|
|
1534
|
-
#endif // SIMSIMD_TARGET_SAPPHIRE
|
|
1535
|
-
#endif // _SIMSIMD_TARGET_X86
|
|
1536
|
-
|
|
1537
|
-
#ifdef __cplusplus
|
|
1538
|
-
}
|
|
1539
|
-
#endif
|
|
1540
|
-
|
|
1541
|
-
#endif
|