usearch 2.21.3 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -24
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +7 -7
- package/c/lib.cpp +32 -14
- package/c/test.c +83 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +235 -52
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -1,1384 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file sparse.h
|
|
3
|
-
* @brief SIMD-accelerated functions for Sparse Vectors.
|
|
4
|
-
* @author Ash Vardanian
|
|
5
|
-
* @date March 21, 2024
|
|
6
|
-
*
|
|
7
|
-
* Contains:
|
|
8
|
-
* - Set Intersection ~ Jaccard Distance
|
|
9
|
-
* - Sparse Dot Products, outputting the count and weighted product
|
|
10
|
-
*
|
|
11
|
-
* For datatypes:
|
|
12
|
-
* - u16: for vocabularies under 64 thousand tokens
|
|
13
|
-
* - u32: for vocabularies under 4 billion tokens
|
|
14
|
-
* - u16 indicies + i16 weights: for weighted word counts
|
|
15
|
-
* - u16 indicies + bf16 weights: for sparse matrices
|
|
16
|
-
*
|
|
17
|
-
* For hardware architectures:
|
|
18
|
-
* - x86: Ice Lake, Turin
|
|
19
|
-
* - Arm: SVE2
|
|
20
|
-
*
|
|
21
|
-
* Interestingly, to implement sparse distances and products, the most important function
|
|
22
|
-
* is analogous to `std::set_intersection`, that outputs the intersection of two sorted
|
|
23
|
-
* sequences. The naive implementation of that function would look like:
|
|
24
|
-
*
|
|
25
|
-
* std::size_t intersection_size = 0;
|
|
26
|
-
* while (i != a_length && j != b_length) {
|
|
27
|
-
* scalar_t ai = a[i], bj = b[j];
|
|
28
|
-
* intersection_size += ai == bj;
|
|
29
|
-
* i += ai < bj;
|
|
30
|
-
* j += ai >= bj;
|
|
31
|
-
* }
|
|
32
|
-
*
|
|
33
|
-
* Assuming we are dealing with sparse arrays, most of the time we are just evaluating
|
|
34
|
-
* branches and skipping entries. So what if we could skip multiple entries at a time
|
|
35
|
-
* searching for the next chunk, where an intersection is possible. For weighted arrays:
|
|
36
|
-
*
|
|
37
|
-
* double product = 0;
|
|
38
|
-
* while (i != a_length && j != b_length) {
|
|
39
|
-
* scalar_t ai = a[i], bj = b[j];
|
|
40
|
-
* product += ai == bj ? a_weights[i] * b_weights[j] : 0;
|
|
41
|
-
* i += ai < bj;
|
|
42
|
-
* j += ai >= bj;
|
|
43
|
-
* }
|
|
44
|
-
*
|
|
45
|
-
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
46
|
-
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
47
|
-
*/
|
|
48
|
-
#ifndef SIMSIMD_SPARSE_H
|
|
49
|
-
#define SIMSIMD_SPARSE_H
|
|
50
|
-
|
|
51
|
-
#include "types.h"
|
|
52
|
-
|
|
53
|
-
#ifdef __cplusplus
|
|
54
|
-
extern "C" {
|
|
55
|
-
#endif
|
|
56
|
-
|
|
57
|
-
/* Implements the serial set intersection algorithm, similar to `std::set_intersection in C++ STL`,
|
|
58
|
-
* but uses clever galloping logic, if the arrays significantly differ in size.
|
|
59
|
-
*/
|
|
60
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_serial( //
|
|
61
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
62
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
63
|
-
simsimd_distance_t *results);
|
|
64
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_serial( //
|
|
65
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
66
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
67
|
-
simsimd_distance_t *results);
|
|
68
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_serial( //
|
|
69
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
70
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
71
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
72
|
-
simsimd_distance_t *results);
|
|
73
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_serial( //
|
|
74
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
75
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
76
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
77
|
-
simsimd_distance_t *results);
|
|
78
|
-
|
|
79
|
-
/* Implements the most naive set intersection algorithm, similar to `std::set_intersection in C++ STL`,
|
|
80
|
-
* naively enumerating the elements of two arrays.
|
|
81
|
-
*/
|
|
82
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_accurate( //
|
|
83
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
84
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
85
|
-
simsimd_distance_t *results);
|
|
86
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_accurate( //
|
|
87
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
88
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
89
|
-
simsimd_distance_t *results);
|
|
90
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_accurate( //
|
|
91
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
92
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
93
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
94
|
-
simsimd_distance_t *results);
|
|
95
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_accurate( //
|
|
96
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
97
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
98
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
99
|
-
simsimd_distance_t *results);
|
|
100
|
-
|
|
101
|
-
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
102
|
-
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
103
|
-
*/
|
|
104
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_sve2( //
|
|
105
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
106
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
107
|
-
simsimd_distance_t *results);
|
|
108
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_sve2( //
|
|
109
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
110
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
111
|
-
simsimd_distance_t *results);
|
|
112
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_sve2( //
|
|
113
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
114
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
115
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
116
|
-
simsimd_distance_t *results);
|
|
117
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_sve2( //
|
|
118
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
119
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
120
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
121
|
-
simsimd_distance_t *results);
|
|
122
|
-
|
|
123
|
-
/* SIMD-powered backends for various generations of AVX512 CPUs.
|
|
124
|
-
* Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
|
|
125
|
-
* Ice Lake, however, is needed even for the most basic kernels to perform integer matching.
|
|
126
|
-
*/
|
|
127
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_ice( //
|
|
128
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
129
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
130
|
-
simsimd_distance_t *results);
|
|
131
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_ice( //
|
|
132
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
133
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
134
|
-
simsimd_distance_t *results);
|
|
135
|
-
|
|
136
|
-
/* SIMD-powered backends for AMD Turin CPUs with cheap VP2INTERSECT instructions.
|
|
137
|
-
* On the Intel side, only mobile Tiger Lake support them, but have prohibitively high latency.
|
|
138
|
-
*/
|
|
139
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_turin( //
|
|
140
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
141
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
142
|
-
simsimd_distance_t *results);
|
|
143
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_turin( //
|
|
144
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
145
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
146
|
-
simsimd_distance_t *results);
|
|
147
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_turin( //
|
|
148
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
149
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
150
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
151
|
-
simsimd_distance_t *results);
|
|
152
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_turin( //
|
|
153
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
154
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
155
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
156
|
-
simsimd_distance_t *results);
|
|
157
|
-
|
|
158
|
-
#define SIMSIMD_MAKE_INTERSECT_LINEAR(name, input_type, counter_type) \
|
|
159
|
-
SIMSIMD_PUBLIC void simsimd_intersect_##input_type##_##name( \
|
|
160
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_size_t a_length, \
|
|
161
|
-
simsimd_size_t b_length, simsimd_distance_t *result) { \
|
|
162
|
-
simsimd_##counter_type##_t intersection_size = 0; \
|
|
163
|
-
simsimd_size_t i = 0, j = 0; \
|
|
164
|
-
while (i != a_length && j != b_length) { \
|
|
165
|
-
simsimd_##input_type##_t ai = a[i]; \
|
|
166
|
-
simsimd_##input_type##_t bj = b[j]; \
|
|
167
|
-
intersection_size += ai == bj; \
|
|
168
|
-
i += ai < bj; \
|
|
169
|
-
j += ai >= bj; \
|
|
170
|
-
} \
|
|
171
|
-
*result = intersection_size; \
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
SIMSIMD_MAKE_INTERSECT_LINEAR(accurate, u16, size) // simsimd_intersect_u16_accurate
|
|
175
|
-
SIMSIMD_MAKE_INTERSECT_LINEAR(accurate, u32, size) // simsimd_intersect_u32_accurate
|
|
176
|
-
|
|
177
|
-
#define SIMSIMD_MAKE_INTERSECT_WEIGHTED(name, variation, input_type, counter_type, weight_type, accumulator_type, \
|
|
178
|
-
load_and_convert) \
|
|
179
|
-
SIMSIMD_PUBLIC void simsimd_##variation##_##input_type##_##name( \
|
|
180
|
-
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, \
|
|
181
|
-
simsimd_##weight_type##_t const *a_weights, simsimd_##weight_type##_t const *b_weights, \
|
|
182
|
-
simsimd_size_t a_length, simsimd_size_t b_length, simsimd_distance_t *results) { \
|
|
183
|
-
simsimd_##counter_type##_t intersection_size = 0; \
|
|
184
|
-
simsimd_##accumulator_type##_t weights_product = 0; \
|
|
185
|
-
simsimd_size_t i = 0, j = 0; \
|
|
186
|
-
while (i != a_length && j != b_length) { \
|
|
187
|
-
simsimd_##input_type##_t ai = a[i]; \
|
|
188
|
-
simsimd_##input_type##_t bj = b[j]; \
|
|
189
|
-
int matches = ai == bj; \
|
|
190
|
-
simsimd_##accumulator_type##_t awi = load_and_convert(a_weights + i); \
|
|
191
|
-
simsimd_##accumulator_type##_t bwi = load_and_convert(b_weights + i); \
|
|
192
|
-
weights_product += matches * awi * bwi; \
|
|
193
|
-
intersection_size += matches; \
|
|
194
|
-
i += ai < bj; \
|
|
195
|
-
j += ai >= bj; \
|
|
196
|
-
} \
|
|
197
|
-
results[0] = intersection_size; \
|
|
198
|
-
results[1] = weights_product; \
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
SIMSIMD_MAKE_INTERSECT_WEIGHTED(accurate, spdot_counts, u16, size, i16, i64,
|
|
202
|
-
SIMSIMD_DEREFERENCE) // simsimd_spdot_counts_u16_accurate
|
|
203
|
-
SIMSIMD_MAKE_INTERSECT_WEIGHTED(accurate, spdot_weights, u16, size, bf16, f64,
|
|
204
|
-
SIMSIMD_BF16_TO_F32) // simsimd_spdot_weights_u16_accurate
|
|
205
|
-
|
|
206
|
-
#define SIMSIMD_MAKE_INTERSECT_GALLOPING(name, input_type, counter_type) \
|
|
207
|
-
SIMSIMD_PUBLIC simsimd_size_t simsimd_galloping_search_##input_type(simsimd_##input_type##_t const *array, \
|
|
208
|
-
simsimd_size_t start, simsimd_size_t length, \
|
|
209
|
-
simsimd_##input_type##_t val) { \
|
|
210
|
-
simsimd_size_t low = start; \
|
|
211
|
-
simsimd_size_t high = start + 1; \
|
|
212
|
-
while (high < length && array[high] < val) { \
|
|
213
|
-
low = high; \
|
|
214
|
-
high = (2 * high < length) ? 2 * high : length; \
|
|
215
|
-
} \
|
|
216
|
-
while (low < high) { \
|
|
217
|
-
simsimd_size_t mid = low + (high - low) / 2; \
|
|
218
|
-
if (array[mid] < val) { low = mid + 1; } \
|
|
219
|
-
else { high = mid; } \
|
|
220
|
-
} \
|
|
221
|
-
return low; \
|
|
222
|
-
} \
|
|
223
|
-
\
|
|
224
|
-
SIMSIMD_PUBLIC void simsimd_intersect_##input_type##_##name( \
|
|
225
|
-
simsimd_##input_type##_t const *shorter, simsimd_##input_type##_t const *longer, \
|
|
226
|
-
simsimd_size_t shorter_length, simsimd_size_t longer_length, simsimd_distance_t *result) { \
|
|
227
|
-
/* Swap arrays if necessary, as we want "longer" to be larger than "shorter" */ \
|
|
228
|
-
if (longer_length < shorter_length) { \
|
|
229
|
-
simsimd_##input_type##_t const *temp = shorter; \
|
|
230
|
-
shorter = longer; \
|
|
231
|
-
longer = temp; \
|
|
232
|
-
simsimd_size_t temp_length = shorter_length; \
|
|
233
|
-
shorter_length = longer_length; \
|
|
234
|
-
longer_length = temp_length; \
|
|
235
|
-
} \
|
|
236
|
-
\
|
|
237
|
-
/* Use the accurate implementation if galloping is not beneficial */ \
|
|
238
|
-
if (longer_length < 64 * shorter_length) { \
|
|
239
|
-
simsimd_intersect_##input_type##_accurate(shorter, longer, shorter_length, longer_length, result); \
|
|
240
|
-
return; \
|
|
241
|
-
} \
|
|
242
|
-
\
|
|
243
|
-
/* Perform galloping, shrinking the target range */ \
|
|
244
|
-
simsimd_##counter_type##_t intersection_size = 0; \
|
|
245
|
-
simsimd_size_t j = 0; \
|
|
246
|
-
for (simsimd_size_t i = 0; i < shorter_length; ++i) { \
|
|
247
|
-
simsimd_##input_type##_t shorter_i = shorter[i]; \
|
|
248
|
-
j = simsimd_galloping_search_##input_type(longer, j, longer_length, shorter_i); \
|
|
249
|
-
if (j < longer_length && longer[j] == shorter_i) { intersection_size++; } \
|
|
250
|
-
} \
|
|
251
|
-
*result = intersection_size; \
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
SIMSIMD_MAKE_INTERSECT_GALLOPING(serial, u16, size) // simsimd_intersect_u16_serial
|
|
255
|
-
SIMSIMD_MAKE_INTERSECT_GALLOPING(serial, u32, size) // simsimd_intersect_u32_serial
|
|
256
|
-
SIMSIMD_MAKE_INTERSECT_WEIGHTED(serial, spdot_counts, u16, size, i16, i32,
|
|
257
|
-
SIMSIMD_DEREFERENCE) // simsimd_spdot_counts_u16_serial
|
|
258
|
-
SIMSIMD_MAKE_INTERSECT_WEIGHTED(serial, spdot_weights, u16, size, bf16, f32,
|
|
259
|
-
SIMSIMD_BF16_TO_F32) // simsimd_spdot_weights_u16_serial
|
|
260
|
-
|
|
261
|
-
/* The AVX-512 implementations are inspired by the "Faster-Than-Native Alternatives
|
|
262
|
-
* for x86 VP2INTERSECT Instructions" paper by Guille Diez-Canas, 2022.
|
|
263
|
-
*
|
|
264
|
-
* https://github.com/mozonaut/vp2intersect
|
|
265
|
-
* https://arxiv.org/pdf/2112.06342.pdf
|
|
266
|
-
*
|
|
267
|
-
* For R&D purposes, it's important to keep the following latencies in mind:
|
|
268
|
-
*
|
|
269
|
-
* - `_mm512_permutex_epi64` - needs F - 3 cycles latency
|
|
270
|
-
* - `_mm512_shuffle_epi8` - needs BW - 1 cycle latency
|
|
271
|
-
* - `_mm512_permutexvar_epi16` - needs BW - 4-6 cycles latency
|
|
272
|
-
* - `_mm512_permutexvar_epi8` - needs VBMI - 3 cycles latency
|
|
273
|
-
*/
|
|
274
|
-
#if _SIMSIMD_TARGET_X86
|
|
275
|
-
#if SIMSIMD_TARGET_ICE
|
|
276
|
-
#pragma GCC push_options
|
|
277
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "lzcnt", "popcnt", "avx512bw", "avx512vbmi2")
|
|
278
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,lzcnt,popcnt,avx512bw,avx512vbmi2"))), \
|
|
279
|
-
apply_to = function)
|
|
280
|
-
|
|
281
|
-
/**
|
|
282
|
-
* @brief Analogous to `_mm512_2intersect_epi16_mask`, but compatible with Ice Lake CPUs,
|
|
283
|
-
* slightly faster than the native Tiger Lake implementation, but returns only one mask.
|
|
284
|
-
*/
|
|
285
|
-
SIMSIMD_INTERNAL simsimd_u32_t _simsimd_intersect_u16x32_ice(__m512i a, __m512i b) {
|
|
286
|
-
__m512i a1 = _mm512_alignr_epi32(a, a, 4);
|
|
287
|
-
__m512i a2 = _mm512_alignr_epi32(a, a, 8);
|
|
288
|
-
__m512i a3 = _mm512_alignr_epi32(a, a, 12);
|
|
289
|
-
|
|
290
|
-
__m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
|
|
291
|
-
__m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
|
|
292
|
-
__m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
|
|
293
|
-
|
|
294
|
-
__m512i b01 = _mm512_shrdi_epi32(b, b, 16);
|
|
295
|
-
__m512i b11 = _mm512_shrdi_epi32(b1, b1, 16);
|
|
296
|
-
__m512i b21 = _mm512_shrdi_epi32(b2, b2, 16);
|
|
297
|
-
__m512i b31 = _mm512_shrdi_epi32(b3, b3, 16);
|
|
298
|
-
|
|
299
|
-
__mmask32 nm00 = _mm512_cmpneq_epi16_mask(a, b);
|
|
300
|
-
__mmask32 nm01 = _mm512_cmpneq_epi16_mask(a1, b);
|
|
301
|
-
__mmask32 nm02 = _mm512_cmpneq_epi16_mask(a2, b);
|
|
302
|
-
__mmask32 nm03 = _mm512_cmpneq_epi16_mask(a3, b);
|
|
303
|
-
|
|
304
|
-
__mmask32 nm10 = _mm512_mask_cmpneq_epi16_mask(nm00, a, b01);
|
|
305
|
-
__mmask32 nm11 = _mm512_mask_cmpneq_epi16_mask(nm01, a1, b01);
|
|
306
|
-
__mmask32 nm12 = _mm512_mask_cmpneq_epi16_mask(nm02, a2, b01);
|
|
307
|
-
__mmask32 nm13 = _mm512_mask_cmpneq_epi16_mask(nm03, a3, b01);
|
|
308
|
-
|
|
309
|
-
__mmask32 nm20 = _mm512_mask_cmpneq_epi16_mask(nm10, a, b1);
|
|
310
|
-
__mmask32 nm21 = _mm512_mask_cmpneq_epi16_mask(nm11, a1, b1);
|
|
311
|
-
__mmask32 nm22 = _mm512_mask_cmpneq_epi16_mask(nm12, a2, b1);
|
|
312
|
-
__mmask32 nm23 = _mm512_mask_cmpneq_epi16_mask(nm13, a3, b1);
|
|
313
|
-
|
|
314
|
-
__mmask32 nm30 = _mm512_mask_cmpneq_epi16_mask(nm20, a, b11);
|
|
315
|
-
__mmask32 nm31 = _mm512_mask_cmpneq_epi16_mask(nm21, a1, b11);
|
|
316
|
-
__mmask32 nm32 = _mm512_mask_cmpneq_epi16_mask(nm22, a2, b11);
|
|
317
|
-
__mmask32 nm33 = _mm512_mask_cmpneq_epi16_mask(nm23, a3, b11);
|
|
318
|
-
|
|
319
|
-
__mmask32 nm40 = _mm512_mask_cmpneq_epi16_mask(nm30, a, b2);
|
|
320
|
-
__mmask32 nm41 = _mm512_mask_cmpneq_epi16_mask(nm31, a1, b2);
|
|
321
|
-
__mmask32 nm42 = _mm512_mask_cmpneq_epi16_mask(nm32, a2, b2);
|
|
322
|
-
__mmask32 nm43 = _mm512_mask_cmpneq_epi16_mask(nm33, a3, b2);
|
|
323
|
-
|
|
324
|
-
__mmask32 nm50 = _mm512_mask_cmpneq_epi16_mask(nm40, a, b21);
|
|
325
|
-
__mmask32 nm51 = _mm512_mask_cmpneq_epi16_mask(nm41, a1, b21);
|
|
326
|
-
__mmask32 nm52 = _mm512_mask_cmpneq_epi16_mask(nm42, a2, b21);
|
|
327
|
-
__mmask32 nm53 = _mm512_mask_cmpneq_epi16_mask(nm43, a3, b21);
|
|
328
|
-
|
|
329
|
-
__mmask32 nm60 = _mm512_mask_cmpneq_epi16_mask(nm50, a, b3);
|
|
330
|
-
__mmask32 nm61 = _mm512_mask_cmpneq_epi16_mask(nm51, a1, b3);
|
|
331
|
-
__mmask32 nm62 = _mm512_mask_cmpneq_epi16_mask(nm52, a2, b3);
|
|
332
|
-
__mmask32 nm63 = _mm512_mask_cmpneq_epi16_mask(nm53, a3, b3);
|
|
333
|
-
|
|
334
|
-
__mmask32 nm70 = _mm512_mask_cmpneq_epi16_mask(nm60, a, b31);
|
|
335
|
-
__mmask32 nm71 = _mm512_mask_cmpneq_epi16_mask(nm61, a1, b31);
|
|
336
|
-
__mmask32 nm72 = _mm512_mask_cmpneq_epi16_mask(nm62, a2, b31);
|
|
337
|
-
__mmask32 nm73 = _mm512_mask_cmpneq_epi16_mask(nm63, a3, b31);
|
|
338
|
-
|
|
339
|
-
return ~(simsimd_u32_t)(nm70 & simsimd_u32_rol(nm71, 8) & simsimd_u32_rol(nm72, 16) & simsimd_u32_ror(nm73, 8));
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
/**
|
|
343
|
-
* @brief Analogous to `_mm512_2intersect_epi32`, but compatible with Ice Lake CPUs,
|
|
344
|
-
* slightly faster than the native Tiger Lake implementation, but returns only one mask.
|
|
345
|
-
*
|
|
346
|
-
* Some latencies to keep in mind:
|
|
347
|
-
*
|
|
348
|
-
* - `_mm512_shuffle_epi32` - "VPSHUFD (ZMM, ZMM, I8)":
|
|
349
|
-
* - 1 cycle latency on Ice Lake: 1*p5
|
|
350
|
-
* - 1 cycle latency on Genoa: 1*FP123
|
|
351
|
-
* - `_mm512_mask_cmpneq_epi32_mask` - "VPCMPD (K, ZMM, ZMM, I8)":
|
|
352
|
-
* - 3 cycle latency on Ice Lake: 1*p5
|
|
353
|
-
* - 1 cycle latency on Genoa: 1*FP01
|
|
354
|
-
* - `_mm512_alignr_epi32` - "VPALIGNR (ZMM, ZMM, ZMM, I8)":
|
|
355
|
-
* - 1 cycle latency on Ice Lake: 1*p5
|
|
356
|
-
* - 2 cycle latency on Genoa: 1*FP12
|
|
357
|
-
* - `_mm512_conflict_epi32` - "VPCONFLICTD (ZMM, ZMM)":
|
|
358
|
-
* - up to 26 cycles latency on Ice Lake: 11*p0+9*p05+17*p5
|
|
359
|
-
* - up to 7 cycle latency on Genoa: 1*FP01+1*FP12
|
|
360
|
-
*/
|
|
361
|
-
SIMSIMD_INTERNAL simsimd_u16_t _simsimd_intersect_u32x16_ice(__m512i a, __m512i b) {
|
|
362
|
-
__m512i a1 = _mm512_alignr_epi32(a, a, 4);
|
|
363
|
-
__m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
|
|
364
|
-
__mmask16 nm00 = _mm512_cmpneq_epi32_mask(a, b);
|
|
365
|
-
|
|
366
|
-
__m512i a2 = _mm512_alignr_epi32(a, a, 8);
|
|
367
|
-
__m512i a3 = _mm512_alignr_epi32(a, a, 12);
|
|
368
|
-
__mmask16 nm01 = _mm512_cmpneq_epi32_mask(a1, b);
|
|
369
|
-
__mmask16 nm02 = _mm512_cmpneq_epi32_mask(a2, b);
|
|
370
|
-
|
|
371
|
-
__mmask16 nm03 = _mm512_cmpneq_epi32_mask(a3, b);
|
|
372
|
-
__mmask16 nm10 = _mm512_mask_cmpneq_epi32_mask(nm00, a, b1);
|
|
373
|
-
__mmask16 nm11 = _mm512_mask_cmpneq_epi32_mask(nm01, a1, b1);
|
|
374
|
-
|
|
375
|
-
__m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
|
|
376
|
-
__mmask16 nm12 = _mm512_mask_cmpneq_epi32_mask(nm02, a2, b1);
|
|
377
|
-
__mmask16 nm13 = _mm512_mask_cmpneq_epi32_mask(nm03, a3, b1);
|
|
378
|
-
__mmask16 nm20 = _mm512_mask_cmpneq_epi32_mask(nm10, a, b2);
|
|
379
|
-
|
|
380
|
-
__m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
|
|
381
|
-
__mmask16 nm21 = _mm512_mask_cmpneq_epi32_mask(nm11, a1, b2);
|
|
382
|
-
__mmask16 nm22 = _mm512_mask_cmpneq_epi32_mask(nm12, a2, b2);
|
|
383
|
-
__mmask16 nm23 = _mm512_mask_cmpneq_epi32_mask(nm13, a3, b2);
|
|
384
|
-
|
|
385
|
-
__mmask16 nm0 = _mm512_mask_cmpneq_epi32_mask(nm20, a, b3);
|
|
386
|
-
__mmask16 nm1 = _mm512_mask_cmpneq_epi32_mask(nm21, a1, b3);
|
|
387
|
-
__mmask16 nm2 = _mm512_mask_cmpneq_epi32_mask(nm22, a2, b3);
|
|
388
|
-
__mmask16 nm3 = _mm512_mask_cmpneq_epi32_mask(nm23, a3, b3);
|
|
389
|
-
|
|
390
|
-
return ~(simsimd_u16_t)(nm0 & simsimd_u16_rol(nm1, 4) & simsimd_u16_rol(nm2, 8) & simsimd_u16_ror(nm3, 4));
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_ice( //
|
|
394
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
395
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
396
|
-
simsimd_distance_t *results) {
|
|
397
|
-
|
|
398
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
399
|
-
if (a_length < 64 && b_length < 64) {
|
|
400
|
-
simsimd_intersect_u16_serial(a, b, a_length, b_length, results);
|
|
401
|
-
return;
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
simsimd_u16_t const *const a_end = a + a_length;
|
|
405
|
-
simsimd_u16_t const *const b_end = b + b_length;
|
|
406
|
-
simsimd_size_t c = 0;
|
|
407
|
-
union vec_t {
|
|
408
|
-
__m512i zmm;
|
|
409
|
-
simsimd_u16_t u16[32];
|
|
410
|
-
simsimd_u8_t u8[64];
|
|
411
|
-
} a_vec, b_vec;
|
|
412
|
-
|
|
413
|
-
while (a + 32 <= a_end && b + 32 <= b_end) {
|
|
414
|
-
a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
|
|
415
|
-
b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
|
|
416
|
-
|
|
417
|
-
// Intersecting registers with `_simsimd_intersect_u16x32_ice` involves a lot of shuffling
|
|
418
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
419
|
-
simsimd_u16_t a_min;
|
|
420
|
-
simsimd_u16_t a_max = a_vec.u16[31];
|
|
421
|
-
simsimd_u16_t b_min = b_vec.u16[0];
|
|
422
|
-
simsimd_u16_t b_max = b_vec.u16[31];
|
|
423
|
-
|
|
424
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
425
|
-
while (a_max < b_min && a + 64 <= a_end) {
|
|
426
|
-
a += 32;
|
|
427
|
-
a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
|
|
428
|
-
a_max = a_vec.u16[31];
|
|
429
|
-
}
|
|
430
|
-
a_min = a_vec.u16[0];
|
|
431
|
-
while (b_max < a_min && b + 64 <= b_end) {
|
|
432
|
-
b += 32;
|
|
433
|
-
b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
|
|
434
|
-
b_max = b_vec.u16[31];
|
|
435
|
-
}
|
|
436
|
-
b_min = b_vec.u16[0];
|
|
437
|
-
|
|
438
|
-
__m512i a_last_broadcasted = _mm512_set1_epi16(*(short const *)&a_max);
|
|
439
|
-
__m512i b_last_broadcasted = _mm512_set1_epi16(*(short const *)&b_max);
|
|
440
|
-
__mmask32 a_step_mask = _mm512_cmple_epu16_mask(a_vec.zmm, b_last_broadcasted);
|
|
441
|
-
__mmask32 b_step_mask = _mm512_cmple_epu16_mask(b_vec.zmm, a_last_broadcasted);
|
|
442
|
-
a += 32 - _lzcnt_u32((simsimd_u32_t)a_step_mask);
|
|
443
|
-
b += 32 - _lzcnt_u32((simsimd_u32_t)b_step_mask);
|
|
444
|
-
|
|
445
|
-
// Now we are likely to have some overlap, so we can intersect the registers
|
|
446
|
-
__mmask32 a_matches = _simsimd_intersect_u16x32_ice(a_vec.zmm, b_vec.zmm);
|
|
447
|
-
|
|
448
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
449
|
-
// but we don't need it here:
|
|
450
|
-
// _mm512_mask_compressstoreu_epi16(c, a_matches, a_vec);
|
|
451
|
-
c += _mm_popcnt_u32(a_matches); // MSVC has no `_popcnt32`
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
|
|
455
|
-
*results += c;
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_ice( //
|
|
459
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
460
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
461
|
-
simsimd_distance_t *results) {
|
|
462
|
-
|
|
463
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
464
|
-
if (a_length < 32 && b_length < 32) {
|
|
465
|
-
simsimd_intersect_u32_serial(a, b, a_length, b_length, results);
|
|
466
|
-
return;
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
simsimd_u32_t const *const a_end = a + a_length;
|
|
470
|
-
simsimd_u32_t const *const b_end = b + b_length;
|
|
471
|
-
simsimd_size_t c = 0;
|
|
472
|
-
union vec_t {
|
|
473
|
-
__m512i zmm;
|
|
474
|
-
simsimd_u32_t u32[16];
|
|
475
|
-
simsimd_u8_t u8[64];
|
|
476
|
-
} a_vec, b_vec;
|
|
477
|
-
|
|
478
|
-
while (a + 16 <= a_end && b + 16 <= b_end) {
|
|
479
|
-
a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
|
|
480
|
-
b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
|
|
481
|
-
|
|
482
|
-
// Intersecting registers with `_simsimd_intersect_u32x16_ice` involves a lot of shuffling
|
|
483
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
484
|
-
simsimd_u32_t a_min;
|
|
485
|
-
simsimd_u32_t a_max = a_vec.u32[15];
|
|
486
|
-
simsimd_u32_t b_min = b_vec.u32[0];
|
|
487
|
-
simsimd_u32_t b_max = b_vec.u32[15];
|
|
488
|
-
|
|
489
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
490
|
-
while (a_max < b_min && a + 32 <= a_end) {
|
|
491
|
-
a += 16;
|
|
492
|
-
a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
|
|
493
|
-
a_max = a_vec.u32[15];
|
|
494
|
-
}
|
|
495
|
-
a_min = a_vec.u32[0];
|
|
496
|
-
while (b_max < a_min && b + 32 <= b_end) {
|
|
497
|
-
b += 16;
|
|
498
|
-
b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
|
|
499
|
-
b_max = b_vec.u32[15];
|
|
500
|
-
}
|
|
501
|
-
b_min = b_vec.u32[0];
|
|
502
|
-
|
|
503
|
-
__m512i a_last_broadcasted = _mm512_set1_epi32(*(int const *)&a_max);
|
|
504
|
-
__m512i b_last_broadcasted = _mm512_set1_epi32(*(int const *)&b_max);
|
|
505
|
-
__mmask16 a_step_mask = _mm512_cmple_epu32_mask(a_vec.zmm, b_last_broadcasted);
|
|
506
|
-
__mmask16 b_step_mask = _mm512_cmple_epu32_mask(b_vec.zmm, a_last_broadcasted);
|
|
507
|
-
a += 32 - _lzcnt_u32((simsimd_u32_t)a_step_mask);
|
|
508
|
-
b += 32 - _lzcnt_u32((simsimd_u32_t)b_step_mask);
|
|
509
|
-
|
|
510
|
-
// Now we are likely to have some overlap, so we can intersect the registers
|
|
511
|
-
__mmask16 a_matches = _simsimd_intersect_u32x16_ice(a_vec.zmm, b_vec.zmm);
|
|
512
|
-
|
|
513
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
514
|
-
// but we don't need it here:
|
|
515
|
-
// _mm512_mask_compressstoreu_epi32(c, a_matches, a_vec);
|
|
516
|
-
c += _mm_popcnt_u32(a_matches); // MSVC has no `_popcnt32`
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
|
|
520
|
-
*results += c;
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
#pragma clang attribute pop
|
|
524
|
-
#pragma GCC pop_options
|
|
525
|
-
#endif // SIMSIMD_TARGET_ICE
|
|
526
|
-
|
|
527
|
-
#if SIMSIMD_TARGET_TURIN
|
|
528
|
-
#pragma GCC push_options
|
|
529
|
-
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi", "bmi2", "lzcnt", "popcnt", "avx512bw", "avx512vbmi2", \
|
|
530
|
-
"avx512bf16", "avx512vnni", "avx512vp2intersect", "avx512dq")
|
|
531
|
-
#pragma clang attribute push( \
|
|
532
|
-
__attribute__((target( \
|
|
533
|
-
"avx2,avx512f,avx512vl,bmi,bmi2,lzcnt,popcnt,avx512bw,avx512vbmi2,avx512bf16,avx512vnni,avx512vp2intersect,avx512dq"))), \
|
|
534
|
-
apply_to = function)
|
|
535
|
-
|
|
536
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_turin( //
|
|
537
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
538
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
539
|
-
simsimd_distance_t *results) {
|
|
540
|
-
|
|
541
|
-
//! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
|
|
542
|
-
//! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
|
|
543
|
-
//! step through 16 entries at a time.
|
|
544
|
-
simsimd_u16_t const *const a_end = a + a_length;
|
|
545
|
-
simsimd_u16_t const *const b_end = b + b_length;
|
|
546
|
-
simsimd_size_t c = 0;
|
|
547
|
-
union vec_t {
|
|
548
|
-
__m256i ymm;
|
|
549
|
-
simsimd_u16_t u16[16];
|
|
550
|
-
simsimd_u8_t u8[32];
|
|
551
|
-
} a_vec, b_vec;
|
|
552
|
-
|
|
553
|
-
// Broadcast index for last element (hoisted outside loop)
|
|
554
|
-
__m256i const last_idx = _mm256_set1_epi16(15);
|
|
555
|
-
while (a + 16 <= a_end && b + 16 <= b_end) {
|
|
556
|
-
a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
|
|
557
|
-
b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
|
|
558
|
-
|
|
559
|
-
// Intersect the registers
|
|
560
|
-
__m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
|
|
561
|
-
__m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
|
|
562
|
-
__mmask16 a_matches_any_in_b, b_matches_any_in_a;
|
|
563
|
-
_mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
|
|
564
|
-
|
|
565
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
566
|
-
// but we don't need it here:
|
|
567
|
-
// _mm512_mask_compressstoreu_epi16(c, a_matches_any_in_b, a_vec);
|
|
568
|
-
c += _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
|
|
569
|
-
|
|
570
|
-
__m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
|
|
571
|
-
__m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
|
|
572
|
-
__mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
|
|
573
|
-
__mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
|
|
574
|
-
a += _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
|
|
575
|
-
b += _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
|
|
579
|
-
*results += c;
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_turin( //
|
|
583
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
584
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
585
|
-
simsimd_distance_t *results) {
|
|
586
|
-
|
|
587
|
-
simsimd_u32_t const *const a_end = a + a_length;
|
|
588
|
-
simsimd_u32_t const *const b_end = b + b_length;
|
|
589
|
-
simsimd_size_t c = 0;
|
|
590
|
-
union vec_t {
|
|
591
|
-
__m512i zmm;
|
|
592
|
-
simsimd_u32_t u32[16];
|
|
593
|
-
simsimd_u8_t u8[64];
|
|
594
|
-
} a_vec, b_vec;
|
|
595
|
-
|
|
596
|
-
// Broadcast index for last element (hoisted outside loop)
|
|
597
|
-
__m512i const last_idx = _mm512_set1_epi32(15);
|
|
598
|
-
while (a + 16 <= a_end && b + 16 <= b_end) {
|
|
599
|
-
a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
|
|
600
|
-
b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
|
|
601
|
-
|
|
602
|
-
// Intersect the registers
|
|
603
|
-
__mmask16 a_matches_any_in_b, b_matches_any_in_a;
|
|
604
|
-
_mm512_2intersect_epi32(a_vec.zmm, b_vec.zmm, &a_matches_any_in_b, &b_matches_any_in_a);
|
|
605
|
-
|
|
606
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
607
|
-
// but we don't need it here:
|
|
608
|
-
// _mm512_mask_compressstoreu_epi32(c, a_matches_any_in_b, a_vec);
|
|
609
|
-
c += _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
|
|
610
|
-
|
|
611
|
-
// Pure SIMD broadcasts - no scalar extraction needed
|
|
612
|
-
__m512i a_last_broadcasted = _mm512_permutexvar_epi32(last_idx, a_vec.zmm);
|
|
613
|
-
__m512i b_last_broadcasted = _mm512_permutexvar_epi32(last_idx, b_vec.zmm);
|
|
614
|
-
__mmask16 a_step_mask = _mm512_cmple_epu32_mask(a_vec.zmm, b_last_broadcasted);
|
|
615
|
-
__mmask16 b_step_mask = _mm512_cmple_epu32_mask(b_vec.zmm, a_last_broadcasted);
|
|
616
|
-
a += _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
|
|
617
|
-
b += _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
|
|
621
|
-
*results += c;
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_turin( //
|
|
625
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
626
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
627
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
628
|
-
simsimd_distance_t *results) {
|
|
629
|
-
|
|
630
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
631
|
-
if (a_length < 64 && b_length < 64) {
|
|
632
|
-
simsimd_spdot_weights_u16_serial(a, b, a_weights, b_weights, a_length, b_length, results);
|
|
633
|
-
return;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
//! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
|
|
637
|
-
//! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
|
|
638
|
-
//! step through 16 entries at a time.
|
|
639
|
-
simsimd_u16_t const *const a_end = a + a_length;
|
|
640
|
-
simsimd_u16_t const *const b_end = b + b_length;
|
|
641
|
-
simsimd_size_t intersection_size = 0;
|
|
642
|
-
union vec_t {
|
|
643
|
-
__m256i ymm;
|
|
644
|
-
__m256 ymmps;
|
|
645
|
-
simsimd_u16_t u16[16];
|
|
646
|
-
simsimd_u8_t u8[32];
|
|
647
|
-
} a_vec, b_vec, product_vec;
|
|
648
|
-
product_vec.ymmps = _mm256_setzero_ps();
|
|
649
|
-
|
|
650
|
-
// Broadcast index for last element (hoisted outside loop)
|
|
651
|
-
__m256i const last_idx = _mm256_set1_epi16(15);
|
|
652
|
-
while (a + 16 <= a_end && b + 16 <= b_end) {
|
|
653
|
-
a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
|
|
654
|
-
b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
|
|
655
|
-
|
|
656
|
-
// Intersecting registers with `_mm512_2intersect_epi16_mask` involves a lot of shuffling
|
|
657
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
658
|
-
simsimd_u16_t a_min;
|
|
659
|
-
simsimd_u16_t a_max = a_vec.u16[15];
|
|
660
|
-
simsimd_u16_t b_min = b_vec.u16[0];
|
|
661
|
-
simsimd_u16_t b_max = b_vec.u16[15];
|
|
662
|
-
|
|
663
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
664
|
-
while (a_max < b_min && a + 32 <= a_end) {
|
|
665
|
-
a += 16, a_weights += 16;
|
|
666
|
-
a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
|
|
667
|
-
a_max = a_vec.u16[15];
|
|
668
|
-
}
|
|
669
|
-
a_min = a_vec.u16[0];
|
|
670
|
-
while (b_max < a_min && b + 32 <= b_end) {
|
|
671
|
-
b += 16, b_weights += 16;
|
|
672
|
-
b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
|
|
673
|
-
b_max = b_vec.u16[15];
|
|
674
|
-
}
|
|
675
|
-
b_min = b_vec.u16[0];
|
|
676
|
-
|
|
677
|
-
// Now we are likely to have some overlap, so we can intersect the registers
|
|
678
|
-
__m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
|
|
679
|
-
__m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
|
|
680
|
-
__mmask16 a_matches_any_in_b, b_matches_any_in_a;
|
|
681
|
-
_mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
|
|
682
|
-
|
|
683
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
684
|
-
// but we don't need it here:
|
|
685
|
-
// _mm512_mask_compressstoreu_epi16(intersection_size, a_matches_any_in_b, a_vec);
|
|
686
|
-
int a_matches_count_in_b = _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
|
|
687
|
-
intersection_size += a_matches_count_in_b;
|
|
688
|
-
|
|
689
|
-
// Load and shift all the relevant weights to the start of the vector before doing the dot product
|
|
690
|
-
if (a_matches_count_in_b) {
|
|
691
|
-
__m256i a_weights_vec = _mm256_lddqu_si256((__m256i const *)a_weights);
|
|
692
|
-
a_weights_vec = _mm256_maskz_compress_epi16(a_matches_any_in_b, a_weights_vec);
|
|
693
|
-
__m256i b_weights_vec = _mm256_lddqu_si256((__m256i const *)b_weights);
|
|
694
|
-
b_weights_vec = _mm256_maskz_compress_epi16(b_matches_any_in_a, b_weights_vec);
|
|
695
|
-
product_vec.ymmps = _mm256_dpbf16_ps(product_vec.ymmps, (__m256bh)a_weights_vec, (__m256bh)b_weights_vec);
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
__m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
|
|
699
|
-
__m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
|
|
700
|
-
__mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
|
|
701
|
-
__mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
|
|
702
|
-
simsimd_size_t a_step = _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
|
|
703
|
-
simsimd_size_t b_step = _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
|
|
704
|
-
a += a_step, a_weights += a_step;
|
|
705
|
-
b += b_step, b_weights += b_step;
|
|
706
|
-
}
|
|
707
|
-
simsimd_spdot_weights_u16_serial(a, b, a_weights, b_weights, a_end - a, b_end - b, results);
|
|
708
|
-
results[0] += intersection_size;
|
|
709
|
-
results[1] += _mm512_reduce_add_ps(_mm512_insertf32x8(_mm512_setzero_ps(), product_vec.ymmps, 0));
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_turin( //
|
|
713
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
714
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
715
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
716
|
-
simsimd_distance_t *results) {
|
|
717
|
-
|
|
718
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
719
|
-
if (a_length < 64 && b_length < 64) {
|
|
720
|
-
simsimd_spdot_counts_u16_serial(a, b, a_weights, b_weights, a_length, b_length, results);
|
|
721
|
-
return;
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
//! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
|
|
725
|
-
//! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
|
|
726
|
-
//! step through 16 entries at a time.
|
|
727
|
-
simsimd_u16_t const *const a_end = a + a_length;
|
|
728
|
-
simsimd_u16_t const *const b_end = b + b_length;
|
|
729
|
-
simsimd_size_t intersection_size = 0;
|
|
730
|
-
union vec_t {
|
|
731
|
-
__m256i ymm;
|
|
732
|
-
simsimd_u16_t u16[16];
|
|
733
|
-
simsimd_u8_t u8[32];
|
|
734
|
-
} a_vec, b_vec, product_vec;
|
|
735
|
-
product_vec.ymm = _mm256_setzero_si256();
|
|
736
|
-
|
|
737
|
-
// Broadcast index for last element (hoisted outside loop)
|
|
738
|
-
__m256i const last_idx = _mm256_set1_epi16(15);
|
|
739
|
-
while (a + 16 <= a_end && b + 16 <= b_end) {
|
|
740
|
-
a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
|
|
741
|
-
b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
|
|
742
|
-
|
|
743
|
-
// Intersecting registers with `_mm512_2intersect_epi16_mask` involves a lot of shuffling
|
|
744
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
745
|
-
simsimd_u16_t a_min;
|
|
746
|
-
simsimd_u16_t a_max = a_vec.u16[15];
|
|
747
|
-
simsimd_u16_t b_min = b_vec.u16[0];
|
|
748
|
-
simsimd_u16_t b_max = b_vec.u16[15];
|
|
749
|
-
|
|
750
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
751
|
-
while (a_max < b_min && a + 32 <= a_end) {
|
|
752
|
-
a += 16, a_weights += 16;
|
|
753
|
-
a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
|
|
754
|
-
a_max = a_vec.u16[15];
|
|
755
|
-
}
|
|
756
|
-
a_min = a_vec.u16[0];
|
|
757
|
-
while (b_max < a_min && b + 32 <= b_end) {
|
|
758
|
-
b += 16, b_weights += 16;
|
|
759
|
-
b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
|
|
760
|
-
b_max = b_vec.u16[15];
|
|
761
|
-
}
|
|
762
|
-
b_min = b_vec.u16[0];
|
|
763
|
-
|
|
764
|
-
// Now we are likely to have some overlap, so we can intersect the registers
|
|
765
|
-
__m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
|
|
766
|
-
__m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
|
|
767
|
-
__mmask16 a_matches_any_in_b, b_matches_any_in_a;
|
|
768
|
-
_mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
|
|
769
|
-
|
|
770
|
-
// The paper also contained a very nice procedure for exporting the matches,
|
|
771
|
-
// but we don't need it here:
|
|
772
|
-
// _mm512_mask_compressstoreu_epi16(intersection_size, a_matches_any_in_b, a_vec);
|
|
773
|
-
int a_matches_count_in_b = _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
|
|
774
|
-
intersection_size += a_matches_count_in_b;
|
|
775
|
-
|
|
776
|
-
// Load and shift all the relevant weights to the start of the vector before doing the dot product
|
|
777
|
-
if (a_matches_count_in_b) {
|
|
778
|
-
__m256i a_weights_vec = _mm256_lddqu_si256((__m256i const *)a_weights);
|
|
779
|
-
a_weights_vec = _mm256_maskz_compress_epi16(a_matches_any_in_b, a_weights_vec);
|
|
780
|
-
__m256i b_weights_vec = _mm256_lddqu_si256((__m256i const *)b_weights);
|
|
781
|
-
b_weights_vec = _mm256_maskz_compress_epi16(b_matches_any_in_a, b_weights_vec);
|
|
782
|
-
product_vec.ymm = _mm256_dpwssds_epi32(product_vec.ymm, a_weights_vec, b_weights_vec);
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
__m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
|
|
786
|
-
__m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
|
|
787
|
-
__mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
|
|
788
|
-
__mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
|
|
789
|
-
simsimd_size_t a_step = _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
|
|
790
|
-
simsimd_size_t b_step = _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
|
|
791
|
-
a += a_step, a_weights += a_step;
|
|
792
|
-
b += b_step, b_weights += b_step;
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
simsimd_spdot_counts_u16_serial(a, b, a_weights, b_weights, a_end - a, b_end - b, results);
|
|
796
|
-
results[0] += intersection_size;
|
|
797
|
-
results[1] += _mm512_reduce_add_epi32(_mm512_inserti64x4(_mm512_setzero_si512(), product_vec.ymm, 0));
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
#pragma clang attribute pop
|
|
801
|
-
#pragma GCC pop_options
|
|
802
|
-
#endif // SIMSIMD_TARGET_TURIN
|
|
803
|
-
#endif // _SIMSIMD_TARGET_X86
|
|
804
|
-
|
|
805
|
-
#if _SIMSIMD_TARGET_ARM
|
|
806
|
-
#if SIMSIMD_TARGET_NEON
|
|
807
|
-
#pragma GCC push_options
|
|
808
|
-
#pragma GCC target("arch=armv8-a")
|
|
809
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8-a"))), apply_to = function)
|
|
810
|
-
|
|
811
|
-
/**
|
|
812
|
-
* @brief Uses `vshrn` to produce a bitmask, similar to `movemask` in SSE.
|
|
813
|
-
* https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
|
|
814
|
-
*/
|
|
815
|
-
SIMSIMD_INTERNAL simsimd_u64_t _simsimd_u8_to_u4_neon(uint8x16_t vec) {
|
|
816
|
-
return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vec), 4)), 0);
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
SIMSIMD_INTERNAL int _simsimd_clz_u64(simsimd_u64_t x) {
|
|
820
|
-
// On GCC and Clang use the builtin, otherwise use the generic implementation
|
|
821
|
-
#if defined(__GNUC__) || defined(__clang__)
|
|
822
|
-
return __builtin_clzll(x);
|
|
823
|
-
#else
|
|
824
|
-
int n = 0;
|
|
825
|
-
while ((x & 0x8000000000000000ull) == 0) n++, x <<= 1;
|
|
826
|
-
return n;
|
|
827
|
-
#endif
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
SIMSIMD_INTERNAL uint32x4_t _simsimd_intersect_u32x4_neon(uint32x4_t a, uint32x4_t b) {
|
|
831
|
-
uint32x4_t b1 = vextq_u32(b, b, 1);
|
|
832
|
-
uint32x4_t b2 = vextq_u32(b, b, 2);
|
|
833
|
-
uint32x4_t b3 = vextq_u32(b, b, 3);
|
|
834
|
-
uint32x4_t nm00 = vceqq_u32(a, b);
|
|
835
|
-
uint32x4_t nm01 = vceqq_u32(a, b1);
|
|
836
|
-
uint32x4_t nm02 = vceqq_u32(a, b2);
|
|
837
|
-
uint32x4_t nm03 = vceqq_u32(a, b3);
|
|
838
|
-
uint32x4_t nm = vorrq_u32(vorrq_u32(nm00, nm01), vorrq_u32(nm02, nm03));
|
|
839
|
-
return nm;
|
|
840
|
-
}
|
|
841
|
-
|
|
842
|
-
SIMSIMD_INTERNAL uint16x8_t _simsimd_intersect_u16x8_neon(uint16x8_t a, uint16x8_t b) {
|
|
843
|
-
uint16x8_t b1 = vextq_u16(b, b, 1);
|
|
844
|
-
uint16x8_t b2 = vextq_u16(b, b, 2);
|
|
845
|
-
uint16x8_t b3 = vextq_u16(b, b, 3);
|
|
846
|
-
uint16x8_t b4 = vextq_u16(b, b, 4);
|
|
847
|
-
uint16x8_t b5 = vextq_u16(b, b, 5);
|
|
848
|
-
uint16x8_t b6 = vextq_u16(b, b, 6);
|
|
849
|
-
uint16x8_t b7 = vextq_u16(b, b, 7);
|
|
850
|
-
uint16x8_t nm00 = vceqq_u16(a, b);
|
|
851
|
-
uint16x8_t nm01 = vceqq_u16(a, b1);
|
|
852
|
-
uint16x8_t nm02 = vceqq_u16(a, b2);
|
|
853
|
-
uint16x8_t nm03 = vceqq_u16(a, b3);
|
|
854
|
-
uint16x8_t nm04 = vceqq_u16(a, b4);
|
|
855
|
-
uint16x8_t nm05 = vceqq_u16(a, b5);
|
|
856
|
-
uint16x8_t nm06 = vceqq_u16(a, b6);
|
|
857
|
-
uint16x8_t nm07 = vceqq_u16(a, b7);
|
|
858
|
-
uint16x8_t nm = vorrq_u16(vorrq_u16(vorrq_u16(nm00, nm01), vorrq_u16(nm02, nm03)),
|
|
859
|
-
vorrq_u16(vorrq_u16(nm04, nm05), vorrq_u16(nm06, nm07)));
|
|
860
|
-
return nm;
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_neon( //
|
|
864
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
865
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
866
|
-
simsimd_distance_t *results) {
|
|
867
|
-
|
|
868
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
869
|
-
if (a_length < 32 && b_length < 32) {
|
|
870
|
-
simsimd_intersect_u16_serial(a, b, a_length, b_length, results);
|
|
871
|
-
return;
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
simsimd_u16_t const *const a_end = a + a_length;
|
|
875
|
-
simsimd_u16_t const *const b_end = b + b_length;
|
|
876
|
-
union vec_t {
|
|
877
|
-
uint16x8_t u16x8;
|
|
878
|
-
simsimd_u16_t u16[8];
|
|
879
|
-
simsimd_u8_t u8[16];
|
|
880
|
-
} a_vec, b_vec, c_counts_vec;
|
|
881
|
-
c_counts_vec.u16x8 = vdupq_n_u16(0);
|
|
882
|
-
|
|
883
|
-
while (a + 8 <= a_end && b + 8 <= b_end) {
|
|
884
|
-
a_vec.u16x8 = vld1q_u16(a);
|
|
885
|
-
b_vec.u16x8 = vld1q_u16(b);
|
|
886
|
-
|
|
887
|
-
// Intersecting registers with `_simsimd_intersect_u16x8_neon` involves a lot of shuffling
|
|
888
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
889
|
-
simsimd_u16_t a_min;
|
|
890
|
-
simsimd_u16_t a_max = a_vec.u16[7];
|
|
891
|
-
simsimd_u16_t b_min = b_vec.u16[0];
|
|
892
|
-
simsimd_u16_t b_max = b_vec.u16[7];
|
|
893
|
-
|
|
894
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
895
|
-
while (a_max < b_min && a + 16 <= a_end) {
|
|
896
|
-
a += 8;
|
|
897
|
-
a_vec.u16x8 = vld1q_u16(a);
|
|
898
|
-
a_max = a_vec.u16[7];
|
|
899
|
-
}
|
|
900
|
-
a_min = a_vec.u16[0];
|
|
901
|
-
while (b_max < a_min && b + 16 <= b_end) {
|
|
902
|
-
b += 8;
|
|
903
|
-
b_vec.u16x8 = vld1q_u16(b);
|
|
904
|
-
b_max = b_vec.u16[7];
|
|
905
|
-
}
|
|
906
|
-
b_min = b_vec.u16[0];
|
|
907
|
-
|
|
908
|
-
// Now we are likely to have some overlap, so we can intersect the registers.
|
|
909
|
-
// We can do it by performing a population count at every cycle, but it's not the cheapest in terms of cycles.
|
|
910
|
-
//
|
|
911
|
-
// simsimd_u64_t a_matches = __builtin_popcountll(
|
|
912
|
-
// _simsimd_u8_to_u4_neon(vreinterpretq_u8_u16(
|
|
913
|
-
// _simsimd_intersect_u16x8_neon(a_vec.u16x8, b_vec.u16x8))));
|
|
914
|
-
// c += a_matches / 8;
|
|
915
|
-
//
|
|
916
|
-
// Alternatively, we can we can transform match-masks into "ones", accumulate them between the cycles,
|
|
917
|
-
// and merge all together in the end.
|
|
918
|
-
uint16x8_t a_matches = _simsimd_intersect_u16x8_neon(a_vec.u16x8, b_vec.u16x8);
|
|
919
|
-
c_counts_vec.u16x8 = vaddq_u16(c_counts_vec.u16x8, vandq_u16(a_matches, vdupq_n_u16(1)));
|
|
920
|
-
|
|
921
|
-
// Counting leading zeros is tricky. On Arm we can use inline Assembly to get the result,
|
|
922
|
-
// but MSVC doesn't support that:
|
|
923
|
-
//
|
|
924
|
-
// SIMSIMD_INTERNAL int _simsimd_clz_u64(simsimd_u64_t value) {
|
|
925
|
-
// simsimd_u64_t result;
|
|
926
|
-
// __asm__("clz %x0, %x1" : "=r"(result) : "r"(value));
|
|
927
|
-
// return (int)result;
|
|
928
|
-
// }
|
|
929
|
-
//
|
|
930
|
-
// Alternatively, we can use the `vclz_u32` NEON intrinsic.
|
|
931
|
-
// It will compute the leading zeros number for both `a_step` and `b_step` in parallel.
|
|
932
|
-
uint16x8_t a_last_broadcasted = vdupq_n_u16(a_max);
|
|
933
|
-
uint16x8_t b_last_broadcasted = vdupq_n_u16(b_max);
|
|
934
|
-
simsimd_u64_t a_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
|
|
935
|
-
vreinterpretq_u8_u16(vcleq_u16(a_vec.u16x8, b_last_broadcasted))));
|
|
936
|
-
simsimd_u64_t b_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
|
|
937
|
-
vreinterpretq_u8_u16(vcleq_u16(b_vec.u16x8, a_last_broadcasted))));
|
|
938
|
-
a += (64 - a_step) / 8;
|
|
939
|
-
b += (64 - b_step) / 8;
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
|
|
943
|
-
*results += vaddvq_u16(c_counts_vec.u16x8);
|
|
944
|
-
}
|
|
945
|
-
|
|
946
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_neon( //
|
|
947
|
-
simsimd_u32_t const *a, simsimd_u32_t const *b, //
|
|
948
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
949
|
-
simsimd_distance_t *results) {
|
|
950
|
-
|
|
951
|
-
// The baseline implementation for very small arrays (2 registers or less) can be quite simple:
|
|
952
|
-
if (a_length < 32 && b_length < 32) {
|
|
953
|
-
simsimd_intersect_u32_serial(a, b, a_length, b_length, results);
|
|
954
|
-
return;
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
simsimd_u32_t const *const a_end = a + a_length;
|
|
958
|
-
simsimd_u32_t const *const b_end = b + b_length;
|
|
959
|
-
union vec_t {
|
|
960
|
-
uint32x4_t u32x4;
|
|
961
|
-
simsimd_u32_t u32[4];
|
|
962
|
-
simsimd_u8_t u8[16];
|
|
963
|
-
} a_vec, b_vec, c_counts_vec;
|
|
964
|
-
c_counts_vec.u32x4 = vdupq_n_u32(0);
|
|
965
|
-
|
|
966
|
-
while (a + 4 <= a_end && b + 4 <= b_end) {
|
|
967
|
-
a_vec.u32x4 = vld1q_u32(a);
|
|
968
|
-
b_vec.u32x4 = vld1q_u32(b);
|
|
969
|
-
|
|
970
|
-
// Intersecting registers with `_simsimd_intersect_u32x4_neon` involves a lot of shuffling
|
|
971
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
972
|
-
simsimd_u32_t a_min;
|
|
973
|
-
simsimd_u32_t a_max = a_vec.u32[3];
|
|
974
|
-
simsimd_u32_t b_min = b_vec.u32[0];
|
|
975
|
-
simsimd_u32_t b_max = b_vec.u32[3];
|
|
976
|
-
|
|
977
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
978
|
-
while (a_max < b_min && a + 8 <= a_end) {
|
|
979
|
-
a += 4;
|
|
980
|
-
a_vec.u32x4 = vld1q_u32(a);
|
|
981
|
-
a_max = a_vec.u32[3];
|
|
982
|
-
}
|
|
983
|
-
a_min = a_vec.u32[0];
|
|
984
|
-
while (b_max < a_min && b + 8 <= b_end) {
|
|
985
|
-
b += 4;
|
|
986
|
-
b_vec.u32x4 = vld1q_u32(b);
|
|
987
|
-
b_max = b_vec.u32[3];
|
|
988
|
-
}
|
|
989
|
-
b_min = b_vec.u32[0];
|
|
990
|
-
|
|
991
|
-
// Now we are likely to have some overlap, so we can intersect the registers
|
|
992
|
-
// We can do it by performing a population count at every cycle, but it's not the cheapest in terms of cycles.
|
|
993
|
-
//
|
|
994
|
-
// simsimd_u64_t a_matches = __builtin_popcountll(
|
|
995
|
-
// _simsimd_u8_to_u4_neon(vreinterpretq_u8_u32(
|
|
996
|
-
// _simsimd_intersect_u32x4_neon(a_vec.u32x4, b_vec.u32x4))));
|
|
997
|
-
// c += a_matches / 16;
|
|
998
|
-
//
|
|
999
|
-
// Alternatively, we can we can transform match-masks into "ones", accumulate them between the cycles,
|
|
1000
|
-
// and merge all together in the end.
|
|
1001
|
-
uint32x4_t a_matches = _simsimd_intersect_u32x4_neon(a_vec.u32x4, b_vec.u32x4);
|
|
1002
|
-
c_counts_vec.u32x4 = vaddq_u32(c_counts_vec.u32x4, vandq_u32(a_matches, vdupq_n_u32(1)));
|
|
1003
|
-
|
|
1004
|
-
uint32x4_t a_last_broadcasted = vdupq_n_u32(a_max);
|
|
1005
|
-
uint32x4_t b_last_broadcasted = vdupq_n_u32(b_max);
|
|
1006
|
-
simsimd_u64_t a_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
|
|
1007
|
-
vreinterpretq_u8_u32(vcleq_u32(a_vec.u32x4, b_last_broadcasted))));
|
|
1008
|
-
simsimd_u64_t b_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
|
|
1009
|
-
vreinterpretq_u8_u32(vcleq_u32(b_vec.u32x4, a_last_broadcasted))));
|
|
1010
|
-
a += (64 - a_step) / 16;
|
|
1011
|
-
b += (64 - b_step) / 16;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
|
|
1015
|
-
*results += vaddvq_u32(c_counts_vec.u32x4);
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
#pragma clang attribute pop
|
|
1019
|
-
#pragma GCC pop_options
|
|
1020
|
-
#endif // SIMSIMD_TARGET_NEON
|
|
1021
|
-
|
|
1022
|
-
/* SVE2 introduces many new integer-oriented instructions, extending some of the NEON functionality
|
|
1023
|
-
* to variable-length SVE registers. Those include "compare multiple" intrinsics:
|
|
1024
|
-
*
|
|
1025
|
-
* - `svmatch[_u16]` that matches each scalar in first vector against all members of a 128-bit lane in the second.
|
|
1026
|
-
* - `svhistcnt[_s32]_z` does something similar, performing an inclusive prefix scan.
|
|
1027
|
-
* - `svtbx[_u16]` does extended table lookup
|
|
1028
|
-
*
|
|
1029
|
-
* Other notable instructions:
|
|
1030
|
-
*
|
|
1031
|
-
* - `DUP`: Broadcast indexed predicate element
|
|
1032
|
-
* https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/DUP--predicate---Broadcast-indexed-predicate-element-?lang=en
|
|
1033
|
-
* - `SCLAMP` and `UCLAMP`: clamp values, i.e. combined min+max
|
|
1034
|
-
* https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/SCLAMP--Signed-clamp-to-minimum-maximum-vector-?lang=en
|
|
1035
|
-
* https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/UCLAMP--Unsigned-clamp-to-minimum-maximum-vector-?lang=en
|
|
1036
|
-
* - `TBLQ`: Table lookup quadword
|
|
1037
|
-
* https://developer.arm.com/documentation/ddi0602/2022-12/SVE-Instructions/TBLQ--Programmable-table-lookup-within-each-quadword-vector-segment--zeroing--?lang=en
|
|
1038
|
-
*
|
|
1039
|
-
* Great resources for SVE2 intrinsics:
|
|
1040
|
-
*
|
|
1041
|
-
* > ARM’s Scalable Vector Extensions: A Critical Look at SVE2 For Integer Workloads
|
|
1042
|
-
* https://gist.github.com/zingaburga/805669eb891c820bd220418ee3f0d6bd
|
|
1043
|
-
*/
|
|
1044
|
-
#if SIMSIMD_TARGET_SVE2
|
|
1045
|
-
#pragma GCC push_options
|
|
1046
|
-
#pragma GCC target("arch=armv8.2-a+sve+sve2")
|
|
1047
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+sve2"))), apply_to = function)
|
|
1048
|
-
|
|
1049
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u16_sve2( //
|
|
1050
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
1051
|
-
simsimd_size_t a_length,
|
|
1052
|
-
simsimd_size_t b_length, //
|
|
1053
|
-
simsimd_distance_t *results) {
|
|
1054
|
-
|
|
1055
|
-
// A single SVE lane is 128 bits wide, so one lane fits 8 values.
|
|
1056
|
-
simsimd_size_t const register_size = svcnth();
|
|
1057
|
-
simsimd_size_t const lanes_count = register_size / 8;
|
|
1058
|
-
simsimd_size_t a_idx = 0, b_idx = 0;
|
|
1059
|
-
simsimd_size_t c = 0;
|
|
1060
|
-
|
|
1061
|
-
while (a_idx < a_length && b_idx < b_length) {
|
|
1062
|
-
// Load `a_member` and broadcast it, load `b_members_vec` from memory
|
|
1063
|
-
svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1064
|
-
svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1065
|
-
svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1066
|
-
svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1067
|
-
|
|
1068
|
-
// Intersecting registers with `svmatch_u16` involves a lot of shuffling
|
|
1069
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
1070
|
-
simsimd_u16_t a_min;
|
|
1071
|
-
simsimd_u16_t a_max = svlastb(a_progress, a_vec);
|
|
1072
|
-
simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
|
|
1073
|
-
simsimd_u16_t b_max = svlastb(b_progress, b_vec);
|
|
1074
|
-
|
|
1075
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
1076
|
-
while (a_max < b_min && (a_idx + register_size) <= a_length) {
|
|
1077
|
-
a_idx += register_size;
|
|
1078
|
-
a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1079
|
-
a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1080
|
-
a_max = svlastb(a_progress, a_vec);
|
|
1081
|
-
}
|
|
1082
|
-
a_min = svlasta(svpfalse_b(), a_vec);
|
|
1083
|
-
while (b_max < a_min && (b_idx + register_size) <= b_length) {
|
|
1084
|
-
b_idx += register_size;
|
|
1085
|
-
b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1086
|
-
b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1087
|
-
b_max = svlastb(b_progress, b_vec);
|
|
1088
|
-
}
|
|
1089
|
-
b_min = svlasta(svpfalse_b(), b_vec);
|
|
1090
|
-
|
|
1091
|
-
// Before we evaluate the intersection size, obfurscating the order in `b_vec`,
|
|
1092
|
-
// let's estimate how much we will need to advance the pointers afterwards.
|
|
1093
|
-
// For that, we don't even need to broadcast the values in SVE, as the whole
|
|
1094
|
-
// register can be compared against a scalar:
|
|
1095
|
-
//
|
|
1096
|
-
// svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
|
|
1097
|
-
// svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
|
|
1098
|
-
svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
|
|
1099
|
-
svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
|
|
1100
|
-
simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
|
|
1101
|
-
simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
|
|
1102
|
-
|
|
1103
|
-
// Compare `a_vec` with each lane of `b_vec`
|
|
1104
|
-
svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
|
|
1105
|
-
for (simsimd_size_t i = 1; i < lanes_count; i++) {
|
|
1106
|
-
b_vec = svext_u16(b_vec, b_vec, 8);
|
|
1107
|
-
equal_mask = svorr_z(svptrue_b16(), equal_mask, svmatch_u16(a_progress, a_vec, b_vec));
|
|
1108
|
-
}
|
|
1109
|
-
simsimd_size_t equal_count = svcntp_b16(svptrue_b16(), equal_mask);
|
|
1110
|
-
|
|
1111
|
-
// Advance
|
|
1112
|
-
a_idx += a_step;
|
|
1113
|
-
b_idx += b_step;
|
|
1114
|
-
c += equal_count;
|
|
1115
|
-
}
|
|
1116
|
-
*results = c;
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
SIMSIMD_PUBLIC void simsimd_intersect_u32_sve2(simsimd_u32_t const *a, simsimd_u32_t const *b, simsimd_size_t a_length,
|
|
1120
|
-
simsimd_size_t b_length, simsimd_distance_t *results) {
|
|
1121
|
-
|
|
1122
|
-
// A single SVE lane is 128 bits wide, so one lane fits 4 values.
|
|
1123
|
-
simsimd_size_t const register_size = svcntw();
|
|
1124
|
-
simsimd_size_t const lanes_count = register_size / 4;
|
|
1125
|
-
simsimd_size_t a_idx = 0, b_idx = 0;
|
|
1126
|
-
simsimd_size_t c = 0;
|
|
1127
|
-
|
|
1128
|
-
while (a_idx < a_length && b_idx < b_length) {
|
|
1129
|
-
// Load `a_member` and broadcast it, load `b_members_vec` from memory
|
|
1130
|
-
svbool_t a_progress = svwhilelt_b32_u64(a_idx, a_length);
|
|
1131
|
-
svbool_t b_progress = svwhilelt_b32_u64(b_idx, b_length);
|
|
1132
|
-
svuint32_t a_vec = svld1_u32(a_progress, a + a_idx);
|
|
1133
|
-
svuint32_t b_vec = svld1_u32(b_progress, b + b_idx);
|
|
1134
|
-
|
|
1135
|
-
// Intersecting registers with `svmatch_u16` involves a lot of shuffling
|
|
1136
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
1137
|
-
simsimd_u32_t a_min;
|
|
1138
|
-
simsimd_u32_t a_max = svlastb(a_progress, a_vec);
|
|
1139
|
-
simsimd_u32_t b_min = svlasta(svpfalse_b(), b_vec);
|
|
1140
|
-
simsimd_u32_t b_max = svlastb(b_progress, b_vec);
|
|
1141
|
-
|
|
1142
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
1143
|
-
while (a_max < b_min && (a_idx + register_size) <= a_length) {
|
|
1144
|
-
a_idx += register_size;
|
|
1145
|
-
a_progress = svwhilelt_b32_u64(a_idx, a_length);
|
|
1146
|
-
a_vec = svld1_u32(a_progress, a + a_idx);
|
|
1147
|
-
a_max = svlastb(a_progress, a_vec);
|
|
1148
|
-
}
|
|
1149
|
-
a_min = svlasta(svpfalse_b(), a_vec);
|
|
1150
|
-
while (b_max < a_min && (b_idx + register_size) <= b_length) {
|
|
1151
|
-
b_idx += register_size;
|
|
1152
|
-
b_progress = svwhilelt_b32_u64(b_idx, b_length);
|
|
1153
|
-
b_vec = svld1_u32(b_progress, b + b_idx);
|
|
1154
|
-
b_max = svlastb(b_progress, b_vec);
|
|
1155
|
-
}
|
|
1156
|
-
b_min = svlasta(svpfalse_b(), b_vec);
|
|
1157
|
-
|
|
1158
|
-
// Before we evaluate the intersection size, obfurscating the order in `b_vec`,
|
|
1159
|
-
// let's estimate how much we will need to advance the pointers afterwards.
|
|
1160
|
-
// For that, we don't even need to broadcast the values in SVE, as the whole
|
|
1161
|
-
// register can be compared against a scalar:
|
|
1162
|
-
//
|
|
1163
|
-
// svuint32_t a_last_broadcasted = svdup_n_u32(a_max);
|
|
1164
|
-
// svuint32_t b_last_broadcasted = svdup_n_u32(b_max);
|
|
1165
|
-
svbool_t a_mask = svcmple_n_u32(a_progress, a_vec, b_max);
|
|
1166
|
-
svbool_t b_mask = svcmple_n_u32(b_progress, b_vec, a_max);
|
|
1167
|
-
simsimd_u64_t a_step = svcntp_b32(a_progress, a_mask);
|
|
1168
|
-
simsimd_u64_t b_step = svcntp_b32(b_progress, b_mask);
|
|
1169
|
-
|
|
1170
|
-
// Comparing `a_vec` with each lane of `b_vec` can't be done with `svmatch`,
|
|
1171
|
-
// the same way as in `simsimd_intersect_u16_sve2`, as that instruction is only
|
|
1172
|
-
// available for 8-bit and 16-bit integers.
|
|
1173
|
-
//
|
|
1174
|
-
// svbool_t equal_mask = svpfalse_b();
|
|
1175
|
-
// for (simsimd_size_t i = 0; i < register_size; i++) {
|
|
1176
|
-
// equal_mask = svorr_z(svptrue_b32(), equal_mask, svcmpeq_u32(a_progress, a_vec, b_vec));
|
|
1177
|
-
// b_vec = svext_u32(b_vec, b_vec, 1);
|
|
1178
|
-
// }
|
|
1179
|
-
// simsimd_size_t equal_count = svcntp_b32(a_progress, equal_mask);
|
|
1180
|
-
//
|
|
1181
|
-
// Alternatively, one can use histogram instructions, like `svhistcnt_u32_z`.
|
|
1182
|
-
// They practically compute the prefix-matching count, which is equivalent to
|
|
1183
|
-
// the lower triangle of the row-major intersection matrix.
|
|
1184
|
-
// To compute the upper triangle, we can reverse (with `svrev_b32`) the order of
|
|
1185
|
-
// elements and repeat the operation, accumulating the results for top and bottom.
|
|
1186
|
-
// Let's look at 4x element registers as an example:
|
|
1187
|
-
//
|
|
1188
|
-
// ⊐ α = {A, B, C, D}, β = {X, Y, Z, W}:
|
|
1189
|
-
//
|
|
1190
|
-
// hist(α, β): hist(α_rev, β_rev):
|
|
1191
|
-
//
|
|
1192
|
-
// X Y Z W W Z Y X
|
|
1193
|
-
// A 1 0 0 0 D 1 0 0 0
|
|
1194
|
-
// B 1 1 0 0 C 1 1 0 0
|
|
1195
|
-
// C 1 1 1 0 B 1 1 1 0
|
|
1196
|
-
// D 1 1 1 1 A 1 1 1 1
|
|
1197
|
-
//
|
|
1198
|
-
svuint32_t hist_lower = svhistcnt_u32_z(a_progress, a_vec, b_vec);
|
|
1199
|
-
svuint32_t a_rev_vec = svrev_u32(a_vec);
|
|
1200
|
-
svuint32_t b_rev_vec = svrev_u32(b_vec);
|
|
1201
|
-
svuint32_t hist_upper = svrev_u32(svhistcnt_u32_z(svptrue_b32(), a_rev_vec, b_rev_vec));
|
|
1202
|
-
svuint32_t hist = svorr_u32_x(a_progress, hist_lower, hist_upper);
|
|
1203
|
-
svbool_t equal_mask = svcmpne_n_u32(a_progress, hist, 0);
|
|
1204
|
-
simsimd_size_t equal_count = svcntp_b32(a_progress, equal_mask);
|
|
1205
|
-
|
|
1206
|
-
// Advance
|
|
1207
|
-
a_idx += a_step;
|
|
1208
|
-
b_idx += b_step;
|
|
1209
|
-
c += equal_count;
|
|
1210
|
-
}
|
|
1211
|
-
*results = c;
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_sve2( //
|
|
1215
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
1216
|
-
simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
|
|
1217
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
1218
|
-
simsimd_distance_t *results) {
|
|
1219
|
-
|
|
1220
|
-
// A single SVE lane is 128 bits wide, so one lane fits 8 values.
|
|
1221
|
-
simsimd_size_t const register_size = svcnth();
|
|
1222
|
-
simsimd_size_t const lanes_count = register_size / 8;
|
|
1223
|
-
simsimd_size_t a_idx = 0, b_idx = 0;
|
|
1224
|
-
svint64_t product_vec = svdupq_n_s64(0, 0);
|
|
1225
|
-
simsimd_size_t intersection_size = 0;
|
|
1226
|
-
|
|
1227
|
-
while (a_idx < a_length && b_idx < b_length) {
|
|
1228
|
-
// Load `a_member` and broadcast it, load `b_members_vec` from memory
|
|
1229
|
-
svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1230
|
-
svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1231
|
-
svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1232
|
-
svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1233
|
-
|
|
1234
|
-
// Intersecting registers with `svmatch_u16` involves a lot of shuffling
|
|
1235
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
1236
|
-
simsimd_u16_t a_min;
|
|
1237
|
-
simsimd_u16_t a_max = svlastb(a_progress, a_vec);
|
|
1238
|
-
simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
|
|
1239
|
-
simsimd_u16_t b_max = svlastb(b_progress, b_vec);
|
|
1240
|
-
|
|
1241
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
1242
|
-
while (a_max < b_min && (a_idx + register_size) <= a_length) {
|
|
1243
|
-
a_idx += register_size;
|
|
1244
|
-
a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1245
|
-
a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1246
|
-
a_max = svlastb(a_progress, a_vec);
|
|
1247
|
-
}
|
|
1248
|
-
a_min = svlasta(svpfalse_b(), a_vec);
|
|
1249
|
-
while (b_max < a_min && (b_idx + register_size) <= b_length) {
|
|
1250
|
-
b_idx += register_size;
|
|
1251
|
-
b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1252
|
-
b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1253
|
-
b_max = svlastb(b_progress, b_vec);
|
|
1254
|
-
}
|
|
1255
|
-
b_min = svlasta(svpfalse_b(), b_vec);
|
|
1256
|
-
|
|
1257
|
-
// Before we evaluate the intersection size, obfurscating the order in `b_vec`,
|
|
1258
|
-
// let's estimate how much we will need to advance the pointers afterwards.
|
|
1259
|
-
// For that, we don't even need to broadcast the values in SVE, as the whole
|
|
1260
|
-
// register can be compared against a scalar:
|
|
1261
|
-
//
|
|
1262
|
-
// svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
|
|
1263
|
-
// svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
|
|
1264
|
-
svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
|
|
1265
|
-
svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
|
|
1266
|
-
simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
|
|
1267
|
-
simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
|
|
1268
|
-
|
|
1269
|
-
// Compare `a_vec` with each lane of `b_vec`
|
|
1270
|
-
svint16_t a_weights_vec = svld1_s16(a_progress, a_weights + a_idx);
|
|
1271
|
-
svint16_t b_weights_vec = svld1_s16(b_progress, b_weights + b_idx);
|
|
1272
|
-
for (simsimd_size_t i = 0; i < lanes_count; i++) {
|
|
1273
|
-
svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
|
|
1274
|
-
svint16_t b_equal_weights_vec = svsel_s16(equal_mask, b_weights_vec, svdup_n_s16(0.f));
|
|
1275
|
-
product_vec = svdot_s64(product_vec, a_weights_vec, b_equal_weights_vec);
|
|
1276
|
-
b_vec = svext_u16(b_vec, b_vec, 8);
|
|
1277
|
-
intersection_size += svcntp_b16(svptrue_b16(), equal_mask);
|
|
1278
|
-
}
|
|
1279
|
-
|
|
1280
|
-
// Advance
|
|
1281
|
-
a_idx += a_step;
|
|
1282
|
-
b_idx += b_step;
|
|
1283
|
-
}
|
|
1284
|
-
results[0] = (simsimd_distance_t)intersection_size;
|
|
1285
|
-
results[1] = svaddv_s64(svptrue_b64(), product_vec);
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
#pragma clang attribute pop
|
|
1289
|
-
#pragma GCC pop_options
|
|
1290
|
-
#endif // SIMSIMD_TARGET_SVE2
|
|
1291
|
-
|
|
1292
|
-
#if SIMSIMD_TARGET_SVE2 && SIMSIMD_TARGET_SVE_BF16
|
|
1293
|
-
#pragma GCC push_options
|
|
1294
|
-
#pragma GCC target("arch=armv8.6-a+sve+sve2+bf16")
|
|
1295
|
-
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+sve+sve2+bf16"))), apply_to = function)
|
|
1296
|
-
|
|
1297
|
-
SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_sve2( //
|
|
1298
|
-
simsimd_u16_t const *a, simsimd_u16_t const *b, //
|
|
1299
|
-
simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
|
|
1300
|
-
simsimd_size_t a_length, simsimd_size_t b_length, //
|
|
1301
|
-
simsimd_distance_t *results) {
|
|
1302
|
-
|
|
1303
|
-
// A single SVE lane is 128 bits wide, so one lane fits 8 values.
|
|
1304
|
-
simsimd_size_t const register_size = svcnth();
|
|
1305
|
-
simsimd_size_t const lanes_count = register_size / 8;
|
|
1306
|
-
simsimd_size_t a_idx = 0, b_idx = 0;
|
|
1307
|
-
svfloat32_t product_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
1308
|
-
simsimd_size_t intersection_size = 0;
|
|
1309
|
-
|
|
1310
|
-
while (a_idx < a_length && b_idx < b_length) {
|
|
1311
|
-
// Load `a_member` and broadcast it, load `b_members_vec` from memory
|
|
1312
|
-
svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1313
|
-
svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1314
|
-
svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1315
|
-
svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1316
|
-
|
|
1317
|
-
// Intersecting registers with `svmatch_u16` involves a lot of shuffling
|
|
1318
|
-
// and comparisons, so we want to avoid it if the slices don't overlap at all..
|
|
1319
|
-
simsimd_u16_t a_min;
|
|
1320
|
-
simsimd_u16_t a_max = svlastb(a_progress, a_vec);
|
|
1321
|
-
simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
|
|
1322
|
-
simsimd_u16_t b_max = svlastb(b_progress, b_vec);
|
|
1323
|
-
|
|
1324
|
-
// If the slices don't overlap, advance the appropriate pointer
|
|
1325
|
-
while (a_max < b_min && (a_idx + register_size) <= a_length) {
|
|
1326
|
-
a_idx += register_size;
|
|
1327
|
-
a_progress = svwhilelt_b16_u64(a_idx, a_length);
|
|
1328
|
-
a_vec = svld1_u16(a_progress, a + a_idx);
|
|
1329
|
-
a_max = svlastb(a_progress, a_vec);
|
|
1330
|
-
}
|
|
1331
|
-
a_min = svlasta(svpfalse_b(), a_vec);
|
|
1332
|
-
while (b_max < a_min && (b_idx + register_size) <= b_length) {
|
|
1333
|
-
b_idx += register_size;
|
|
1334
|
-
b_progress = svwhilelt_b16_u64(b_idx, b_length);
|
|
1335
|
-
b_vec = svld1_u16(b_progress, b + b_idx);
|
|
1336
|
-
b_max = svlastb(b_progress, b_vec);
|
|
1337
|
-
}
|
|
1338
|
-
b_min = svlasta(svpfalse_b(), b_vec);
|
|
1339
|
-
|
|
1340
|
-
// Before we evaluate the intersection size, obfurscating the order in `b_vec`,
|
|
1341
|
-
// let's estimate how much we will need to advance the pointers afterwards.
|
|
1342
|
-
// For that, we don't even need to broadcast the values in SVE, as the whole
|
|
1343
|
-
// register can be compared against a scalar:
|
|
1344
|
-
//
|
|
1345
|
-
// svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
|
|
1346
|
-
// svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
|
|
1347
|
-
svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
|
|
1348
|
-
svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
|
|
1349
|
-
simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
|
|
1350
|
-
simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
|
|
1351
|
-
|
|
1352
|
-
// Compare `a_vec` with each lane of `b_vec`
|
|
1353
|
-
svbfloat16_t a_weights_vec = svld1_bf16(a_progress, (__bf16 const *)a_weights + a_idx);
|
|
1354
|
-
svbfloat16_t b_weights_vec = svld1_bf16(b_progress, (__bf16 const *)b_weights + b_idx);
|
|
1355
|
-
for (simsimd_size_t i = 0; i < lanes_count; i++) {
|
|
1356
|
-
svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
|
|
1357
|
-
//! The `svsel_bf16` intrinsic is broken in many compilers, not returning the correct type.
|
|
1358
|
-
//! So we reinterprete floats as integers and apply `svsel_s16`, but the `svreinterpret_s16_bs16`
|
|
1359
|
-
//! and `svreinterpret_bf16_s16` are not always properly defined!
|
|
1360
|
-
svint16_t b_equal_weights_vec =
|
|
1361
|
-
svsel_s16(equal_mask, svreinterpret_s16_bf16(b_weights_vec), svdup_n_s16(0));
|
|
1362
|
-
product_vec = svbfdot_f32(product_vec, a_weights_vec, svreinterpret_bf16_s16(b_equal_weights_vec));
|
|
1363
|
-
b_vec = svext_u16(b_vec, b_vec, 8);
|
|
1364
|
-
intersection_size += svcntp_b16(svptrue_b16(), equal_mask);
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
// Advance
|
|
1368
|
-
a_idx += a_step;
|
|
1369
|
-
b_idx += b_step;
|
|
1370
|
-
}
|
|
1371
|
-
results[0] = (simsimd_distance_t)intersection_size;
|
|
1372
|
-
results[1] = svaddv_f32(svptrue_b32(), product_vec);
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
#pragma clang attribute pop
|
|
1376
|
-
#pragma GCC pop_options
|
|
1377
|
-
#endif // SIMSIMD_TARGET_SVE2 && SIMSIMD_TARGET_SVE_BF16
|
|
1378
|
-
#endif // _SIMSIMD_TARGET_ARM
|
|
1379
|
-
|
|
1380
|
-
#ifdef __cplusplus
|
|
1381
|
-
}
|
|
1382
|
-
#endif
|
|
1383
|
-
|
|
1384
|
-
#endif
|