usearch 2.21.3 → 2.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (602) hide show
  1. package/README.md +24 -24
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +7 -7
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +83 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +235 -52
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/dist/cjs/usearch.d.ts +8 -3
  16. package/javascript/dist/cjs/usearch.js +12 -4
  17. package/javascript/dist/esm/usearch.d.ts +8 -3
  18. package/javascript/dist/esm/usearch.js +12 -4
  19. package/javascript/lib.cpp +17 -0
  20. package/javascript/usearch.ts +14 -6
  21. package/numkong/.clang-tidy +30 -0
  22. package/{simsimd → numkong}/.cmake-format.py +18 -0
  23. package/numkong/.editorconfig +12 -0
  24. package/numkong/.git-blame-ignore-revs +123 -0
  25. package/numkong/.mailmap +15 -0
  26. package/numkong/.spi.yml +11 -0
  27. package/numkong/.swift-format +14 -0
  28. package/numkong/CITATION.cff +81 -0
  29. package/numkong/CONTRIBUTING.md +500 -0
  30. package/numkong/Cargo.toml +54 -0
  31. package/numkong/MANIFEST.in +8 -0
  32. package/numkong/Package.swift +84 -0
  33. package/numkong/README.md +569 -0
  34. package/numkong/bench/README.md +217 -0
  35. package/numkong/bench/bench-browser.html +323 -0
  36. package/numkong/bench/bench.cpp +344 -0
  37. package/numkong/bench/bench.hpp +816 -0
  38. package/numkong/bench/bench.js +148 -0
  39. package/numkong/bench/bench.mjs +616 -0
  40. package/numkong/bench/bench_attention.cpp +113 -0
  41. package/numkong/bench/bench_cast.cpp +264 -0
  42. package/numkong/bench/bench_cross_amx.cpp +62 -0
  43. package/numkong/bench/bench_cross_arm.cpp +245 -0
  44. package/numkong/bench/bench_cross_blas.cpp +309 -0
  45. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  46. package/numkong/bench/bench_cross_power.cpp +84 -0
  47. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  48. package/numkong/bench/bench_cross_serial.cpp +151 -0
  49. package/numkong/bench/bench_cross_sme.cpp +163 -0
  50. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  51. package/numkong/bench/bench_cross_x86.cpp +455 -0
  52. package/numkong/bench/bench_curved.cpp +210 -0
  53. package/numkong/bench/bench_dot.cpp +315 -0
  54. package/numkong/bench/bench_each.cpp +503 -0
  55. package/numkong/bench/bench_geospatial.cpp +112 -0
  56. package/numkong/bench/bench_maxsim.cpp +154 -0
  57. package/numkong/bench/bench_mesh.cpp +148 -0
  58. package/numkong/bench/bench_probability.cpp +60 -0
  59. package/numkong/bench/bench_reduce.cpp +371 -0
  60. package/numkong/bench/bench_set.cpp +73 -0
  61. package/numkong/bench/bench_sparse.cpp +198 -0
  62. package/numkong/bench/bench_spatial.cpp +347 -0
  63. package/numkong/bench/bench_trigonometry.cpp +137 -0
  64. package/numkong/bench/swift/Bench.swift +506 -0
  65. package/numkong/binding.gyp +169 -0
  66. package/numkong/build.rs +484 -0
  67. package/numkong/c/dispatch.h +512 -0
  68. package/numkong/c/dispatch_bf16.c +447 -0
  69. package/numkong/c/dispatch_bf16c.c +52 -0
  70. package/numkong/c/dispatch_e2m3.c +296 -0
  71. package/numkong/c/dispatch_e3m2.c +284 -0
  72. package/numkong/c/dispatch_e4m3.c +318 -0
  73. package/numkong/c/dispatch_e5m2.c +316 -0
  74. package/numkong/c/dispatch_f16.c +452 -0
  75. package/numkong/c/dispatch_f16c.c +58 -0
  76. package/numkong/c/dispatch_f32.c +434 -0
  77. package/numkong/c/dispatch_f32c.c +99 -0
  78. package/numkong/c/dispatch_f64.c +348 -0
  79. package/numkong/c/dispatch_f64c.c +98 -0
  80. package/numkong/c/dispatch_i16.c +96 -0
  81. package/numkong/c/dispatch_i32.c +89 -0
  82. package/numkong/c/dispatch_i4.c +153 -0
  83. package/numkong/c/dispatch_i64.c +86 -0
  84. package/numkong/c/dispatch_i8.c +348 -0
  85. package/numkong/c/dispatch_other.c +348 -0
  86. package/numkong/c/dispatch_u1.c +193 -0
  87. package/numkong/c/dispatch_u16.c +124 -0
  88. package/numkong/c/dispatch_u32.c +118 -0
  89. package/numkong/c/dispatch_u4.c +153 -0
  90. package/numkong/c/dispatch_u64.c +102 -0
  91. package/numkong/c/dispatch_u8.c +364 -0
  92. package/numkong/c/numkong.c +940 -0
  93. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  94. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  95. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  96. package/numkong/cmake/nk_math_lib.cmake +46 -0
  97. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  99. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  100. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  101. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  102. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  103. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  104. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  105. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  106. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  107. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  108. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  109. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  110. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  111. package/numkong/include/README.md +643 -0
  112. package/numkong/include/module.modulemap +129 -0
  113. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  114. package/numkong/include/numkong/attention/sme.h +2075 -0
  115. package/numkong/include/numkong/attention.h +49 -0
  116. package/numkong/include/numkong/capabilities.h +930 -0
  117. package/numkong/include/numkong/cast/README.md +262 -0
  118. package/numkong/include/numkong/cast/diamond.h +64 -0
  119. package/numkong/include/numkong/cast/haswell.h +823 -0
  120. package/numkong/include/numkong/cast/icelake.h +475 -0
  121. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  122. package/numkong/include/numkong/cast/neon.h +1174 -0
  123. package/numkong/include/numkong/cast/powervsx.h +449 -0
  124. package/numkong/include/numkong/cast/rvv.h +970 -0
  125. package/numkong/include/numkong/cast/sapphire.h +262 -0
  126. package/numkong/include/numkong/cast/serial.h +2336 -0
  127. package/numkong/include/numkong/cast/skylake.h +915 -0
  128. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  129. package/numkong/include/numkong/cast.h +254 -0
  130. package/numkong/include/numkong/cast.hpp +44 -0
  131. package/numkong/include/numkong/curved/README.md +223 -0
  132. package/numkong/include/numkong/curved/genoa.h +182 -0
  133. package/numkong/include/numkong/curved/haswell.h +276 -0
  134. package/numkong/include/numkong/curved/neon.h +329 -0
  135. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  136. package/numkong/include/numkong/curved/rvv.h +305 -0
  137. package/numkong/include/numkong/curved/serial.h +207 -0
  138. package/numkong/include/numkong/curved/skylake.h +457 -0
  139. package/numkong/include/numkong/curved/smef64.h +530 -0
  140. package/numkong/include/numkong/curved.h +513 -0
  141. package/numkong/include/numkong/curved.hpp +144 -0
  142. package/numkong/include/numkong/dot/README.md +442 -0
  143. package/numkong/include/numkong/dot/alder.h +563 -0
  144. package/numkong/include/numkong/dot/diamond.h +158 -0
  145. package/numkong/include/numkong/dot/genoa.h +289 -0
  146. package/numkong/include/numkong/dot/haswell.h +1723 -0
  147. package/numkong/include/numkong/dot/icelake.h +997 -0
  148. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  149. package/numkong/include/numkong/dot/neon.h +869 -0
  150. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  151. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  152. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  153. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  154. package/numkong/include/numkong/dot/powervsx.h +752 -0
  155. package/numkong/include/numkong/dot/rvv.h +722 -0
  156. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  157. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  158. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  159. package/numkong/include/numkong/dot/sapphire.h +141 -0
  160. package/numkong/include/numkong/dot/serial.h +874 -0
  161. package/numkong/include/numkong/dot/sierra.h +405 -0
  162. package/numkong/include/numkong/dot/skylake.h +1116 -0
  163. package/numkong/include/numkong/dot/sve.h +420 -0
  164. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  165. package/numkong/include/numkong/dot/svehalf.h +168 -0
  166. package/numkong/include/numkong/dot/svesdot.h +90 -0
  167. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  168. package/numkong/include/numkong/dot.h +1136 -0
  169. package/numkong/include/numkong/dot.hpp +94 -0
  170. package/numkong/include/numkong/dots/README.md +496 -0
  171. package/numkong/include/numkong/dots/alder.h +117 -0
  172. package/numkong/include/numkong/dots/diamond.h +86 -0
  173. package/numkong/include/numkong/dots/genoa.h +100 -0
  174. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  175. package/numkong/include/numkong/dots/haswell.h +310 -0
  176. package/numkong/include/numkong/dots/icelake.h +180 -0
  177. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  178. package/numkong/include/numkong/dots/neon.h +123 -0
  179. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  180. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  181. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  182. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  183. package/numkong/include/numkong/dots/powervsx.h +194 -0
  184. package/numkong/include/numkong/dots/rvv.h +2593 -0
  185. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  186. package/numkong/include/numkong/dots/serial.h +2867 -0
  187. package/numkong/include/numkong/dots/sierra.h +100 -0
  188. package/numkong/include/numkong/dots/skylake.h +207 -0
  189. package/numkong/include/numkong/dots/sme.h +5041 -0
  190. package/numkong/include/numkong/dots/smebi32.h +477 -0
  191. package/numkong/include/numkong/dots/smef64.h +1328 -0
  192. package/numkong/include/numkong/dots/smehalf.h +47 -0
  193. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  194. package/numkong/include/numkong/dots.h +3110 -0
  195. package/numkong/include/numkong/dots.hpp +641 -0
  196. package/numkong/include/numkong/each/README.md +465 -0
  197. package/numkong/include/numkong/each/haswell.h +1658 -0
  198. package/numkong/include/numkong/each/icelake.h +272 -0
  199. package/numkong/include/numkong/each/neon.h +1138 -0
  200. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  201. package/numkong/include/numkong/each/neonhalf.h +318 -0
  202. package/numkong/include/numkong/each/rvv.h +1121 -0
  203. package/numkong/include/numkong/each/sapphire.h +343 -0
  204. package/numkong/include/numkong/each/serial.h +282 -0
  205. package/numkong/include/numkong/each/skylake.h +1562 -0
  206. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  207. package/numkong/include/numkong/each.h +2232 -0
  208. package/numkong/include/numkong/each.hpp +434 -0
  209. package/numkong/include/numkong/geospatial/README.md +147 -0
  210. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  211. package/numkong/include/numkong/geospatial/neon.h +615 -0
  212. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  213. package/numkong/include/numkong/geospatial/serial.h +309 -0
  214. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  215. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  216. package/numkong/include/numkong/geospatial.h +453 -0
  217. package/numkong/include/numkong/geospatial.hpp +235 -0
  218. package/numkong/include/numkong/matrix.hpp +336 -0
  219. package/numkong/include/numkong/maxsim/README.md +187 -0
  220. package/numkong/include/numkong/maxsim/alder.h +511 -0
  221. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  222. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  223. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  224. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  225. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  226. package/numkong/include/numkong/maxsim/serial.h +490 -0
  227. package/numkong/include/numkong/maxsim/sme.h +993 -0
  228. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  229. package/numkong/include/numkong/maxsim.h +571 -0
  230. package/numkong/include/numkong/maxsim.hpp +133 -0
  231. package/numkong/include/numkong/mesh/README.md +219 -0
  232. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  233. package/numkong/include/numkong/mesh/neon.h +1821 -0
  234. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  235. package/numkong/include/numkong/mesh/rvv.h +936 -0
  236. package/numkong/include/numkong/mesh/serial.h +721 -0
  237. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  238. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  239. package/numkong/include/numkong/mesh.h +666 -0
  240. package/numkong/include/numkong/mesh.hpp +701 -0
  241. package/numkong/include/numkong/numkong.h +78 -0
  242. package/numkong/include/numkong/numkong.hpp +58 -0
  243. package/numkong/include/numkong/probability/README.md +177 -0
  244. package/numkong/include/numkong/probability/haswell.h +267 -0
  245. package/numkong/include/numkong/probability/neon.h +255 -0
  246. package/numkong/include/numkong/probability/rvv.h +414 -0
  247. package/numkong/include/numkong/probability/serial.h +181 -0
  248. package/numkong/include/numkong/probability/skylake.h +324 -0
  249. package/numkong/include/numkong/probability.h +380 -0
  250. package/numkong/include/numkong/probability.hpp +120 -0
  251. package/numkong/include/numkong/random.h +50 -0
  252. package/numkong/include/numkong/random.hpp +285 -0
  253. package/numkong/include/numkong/reduce/README.md +552 -0
  254. package/numkong/include/numkong/reduce/alder.h +636 -0
  255. package/numkong/include/numkong/reduce/genoa.h +201 -0
  256. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  257. package/numkong/include/numkong/reduce/icelake.h +549 -0
  258. package/numkong/include/numkong/reduce/neon.h +3980 -0
  259. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  260. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  261. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  262. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  263. package/numkong/include/numkong/reduce/serial.h +753 -0
  264. package/numkong/include/numkong/reduce/sierra.h +338 -0
  265. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  266. package/numkong/include/numkong/reduce/sve.h +52 -0
  267. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  268. package/numkong/include/numkong/reduce.h +1582 -0
  269. package/numkong/include/numkong/reduce.hpp +778 -0
  270. package/numkong/include/numkong/scalar/README.md +89 -0
  271. package/numkong/include/numkong/scalar/haswell.h +122 -0
  272. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  273. package/numkong/include/numkong/scalar/neon.h +122 -0
  274. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  275. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  276. package/numkong/include/numkong/scalar/rvv.h +211 -0
  277. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  278. package/numkong/include/numkong/scalar/serial.h +332 -0
  279. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  280. package/numkong/include/numkong/scalar.h +696 -0
  281. package/numkong/include/numkong/set/README.md +179 -0
  282. package/numkong/include/numkong/set/haswell.h +334 -0
  283. package/numkong/include/numkong/set/icelake.h +485 -0
  284. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  285. package/numkong/include/numkong/set/neon.h +363 -0
  286. package/numkong/include/numkong/set/powervsx.h +326 -0
  287. package/numkong/include/numkong/set/rvv.h +226 -0
  288. package/numkong/include/numkong/set/rvvbb.h +117 -0
  289. package/numkong/include/numkong/set/serial.h +174 -0
  290. package/numkong/include/numkong/set/sve.h +187 -0
  291. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  292. package/numkong/include/numkong/set.h +471 -0
  293. package/numkong/include/numkong/set.hpp +114 -0
  294. package/numkong/include/numkong/sets/README.md +149 -0
  295. package/numkong/include/numkong/sets/haswell.h +63 -0
  296. package/numkong/include/numkong/sets/icelake.h +66 -0
  297. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  298. package/numkong/include/numkong/sets/neon.h +61 -0
  299. package/numkong/include/numkong/sets/powervsx.h +65 -0
  300. package/numkong/include/numkong/sets/serial.h +43 -0
  301. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  302. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  303. package/numkong/include/numkong/sets.h +382 -0
  304. package/numkong/include/numkong/sparse/README.md +156 -0
  305. package/numkong/include/numkong/sparse/icelake.h +463 -0
  306. package/numkong/include/numkong/sparse/neon.h +288 -0
  307. package/numkong/include/numkong/sparse/serial.h +117 -0
  308. package/numkong/include/numkong/sparse/sve2.h +504 -0
  309. package/numkong/include/numkong/sparse/turin.h +322 -0
  310. package/numkong/include/numkong/sparse.h +363 -0
  311. package/numkong/include/numkong/sparse.hpp +113 -0
  312. package/numkong/include/numkong/spatial/README.md +439 -0
  313. package/numkong/include/numkong/spatial/alder.h +604 -0
  314. package/numkong/include/numkong/spatial/diamond.h +240 -0
  315. package/numkong/include/numkong/spatial/genoa.h +222 -0
  316. package/numkong/include/numkong/spatial/haswell.h +979 -0
  317. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  318. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  319. package/numkong/include/numkong/spatial/neon.h +846 -0
  320. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  321. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  322. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  323. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  324. package/numkong/include/numkong/spatial/rvv.h +991 -0
  325. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  326. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  327. package/numkong/include/numkong/spatial/serial.h +347 -0
  328. package/numkong/include/numkong/spatial/sierra.h +516 -0
  329. package/numkong/include/numkong/spatial/skylake.h +605 -0
  330. package/numkong/include/numkong/spatial/sve.h +246 -0
  331. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  332. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  333. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  334. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  335. package/numkong/include/numkong/spatial.h +1681 -0
  336. package/numkong/include/numkong/spatial.hpp +183 -0
  337. package/numkong/include/numkong/spatials/README.md +580 -0
  338. package/numkong/include/numkong/spatials/alder.h +94 -0
  339. package/numkong/include/numkong/spatials/diamond.h +82 -0
  340. package/numkong/include/numkong/spatials/genoa.h +94 -0
  341. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  342. package/numkong/include/numkong/spatials/haswell.h +219 -0
  343. package/numkong/include/numkong/spatials/icelake.h +113 -0
  344. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  345. package/numkong/include/numkong/spatials/neon.h +109 -0
  346. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  347. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  348. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  349. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  350. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  351. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  352. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  353. package/numkong/include/numkong/spatials/serial.h +248 -0
  354. package/numkong/include/numkong/spatials/sierra.h +96 -0
  355. package/numkong/include/numkong/spatials/skylake.h +184 -0
  356. package/numkong/include/numkong/spatials/sme.h +1927 -0
  357. package/numkong/include/numkong/spatials/smef64.h +483 -0
  358. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  359. package/numkong/include/numkong/spatials.h +3374 -0
  360. package/numkong/include/numkong/spatials.hpp +509 -0
  361. package/numkong/include/numkong/tensor.hpp +1981 -0
  362. package/numkong/include/numkong/trigonometry/README.md +184 -0
  363. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  364. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  365. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  366. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  367. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  368. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  369. package/numkong/include/numkong/trigonometry.h +467 -0
  370. package/numkong/include/numkong/trigonometry.hpp +166 -0
  371. package/numkong/include/numkong/types.h +1695 -0
  372. package/numkong/include/numkong/types.hpp +6027 -0
  373. package/numkong/include/numkong/vector.hpp +766 -0
  374. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  375. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  376. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  377. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  378. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  380. package/numkong/javascript/README.md +254 -0
  381. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  382. package/numkong/javascript/numkong-browser.ts +40 -0
  383. package/numkong/javascript/numkong-wasm.ts +750 -0
  384. package/numkong/javascript/numkong.c +712 -0
  385. package/numkong/javascript/numkong.ts +613 -0
  386. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  387. package/numkong/javascript/tsconfig-cjs.json +13 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  389. package/numkong/javascript/types.ts +784 -0
  390. package/numkong/numkong.gypi +104 -0
  391. package/numkong/package.json +108 -0
  392. package/numkong/probes/arm_neon.c +8 -0
  393. package/numkong/probes/arm_neon_bfdot.c +9 -0
  394. package/numkong/probes/arm_neon_fhm.c +9 -0
  395. package/numkong/probes/arm_neon_half.c +8 -0
  396. package/numkong/probes/arm_neon_sdot.c +9 -0
  397. package/numkong/probes/arm_neonfp8.c +9 -0
  398. package/numkong/probes/arm_sme.c +16 -0
  399. package/numkong/probes/arm_sme2.c +16 -0
  400. package/numkong/probes/arm_sme2p1.c +16 -0
  401. package/numkong/probes/arm_sme_bf16.c +16 -0
  402. package/numkong/probes/arm_sme_bi32.c +16 -0
  403. package/numkong/probes/arm_sme_f64.c +16 -0
  404. package/numkong/probes/arm_sme_fa64.c +14 -0
  405. package/numkong/probes/arm_sme_half.c +16 -0
  406. package/numkong/probes/arm_sme_lut2.c +15 -0
  407. package/numkong/probes/arm_sve.c +18 -0
  408. package/numkong/probes/arm_sve2.c +20 -0
  409. package/numkong/probes/arm_sve2p1.c +18 -0
  410. package/numkong/probes/arm_sve_bfdot.c +20 -0
  411. package/numkong/probes/arm_sve_half.c +18 -0
  412. package/numkong/probes/arm_sve_sdot.c +21 -0
  413. package/numkong/probes/loongarch_lasx.c +12 -0
  414. package/numkong/probes/power_vsx.c +12 -0
  415. package/numkong/probes/probe.js +127 -0
  416. package/numkong/probes/riscv_rvv.c +14 -0
  417. package/numkong/probes/riscv_rvv_bb.c +15 -0
  418. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  419. package/numkong/probes/riscv_rvv_half.c +14 -0
  420. package/numkong/probes/wasm_v128relaxed.c +11 -0
  421. package/numkong/probes/x86_alder.c +17 -0
  422. package/numkong/probes/x86_diamond.c +17 -0
  423. package/numkong/probes/x86_genoa.c +17 -0
  424. package/numkong/probes/x86_graniteamx.c +19 -0
  425. package/numkong/probes/x86_haswell.c +11 -0
  426. package/numkong/probes/x86_icelake.c +17 -0
  427. package/numkong/probes/x86_sapphire.c +16 -0
  428. package/numkong/probes/x86_sapphireamx.c +18 -0
  429. package/numkong/probes/x86_sierra.c +17 -0
  430. package/numkong/probes/x86_skylake.c +15 -0
  431. package/numkong/probes/x86_turin.c +17 -0
  432. package/numkong/rust/README.md +698 -0
  433. package/numkong/rust/capabilities.rs +96 -0
  434. package/numkong/rust/cast.rs +314 -0
  435. package/numkong/rust/curved.rs +391 -0
  436. package/numkong/rust/each.rs +2721 -0
  437. package/numkong/rust/geospatial.rs +343 -0
  438. package/numkong/rust/matrix.rs +5424 -0
  439. package/numkong/rust/maxsim.rs +540 -0
  440. package/numkong/rust/mesh.rs +837 -0
  441. package/numkong/rust/numkong.rs +409 -0
  442. package/numkong/rust/probability.rs +272 -0
  443. package/numkong/rust/reduce.rs +1249 -0
  444. package/numkong/rust/set.rs +194 -0
  445. package/numkong/rust/sparse.rs +513 -0
  446. package/numkong/rust/spatial.rs +1815 -0
  447. package/numkong/rust/tensor.rs +7565 -0
  448. package/numkong/rust/types.rs +3666 -0
  449. package/numkong/rust/vector.rs +1899 -0
  450. package/numkong/rustfmt.toml +2 -0
  451. package/numkong/swift/Matrix.swift +1388 -0
  452. package/numkong/swift/MaxSim.swift +151 -0
  453. package/numkong/swift/NumKong.swift +283 -0
  454. package/numkong/swift/Numerics.swift +329 -0
  455. package/numkong/swift/README.md +433 -0
  456. package/numkong/swift/Scalar.swift +741 -0
  457. package/numkong/swift/Tensor.swift +209 -0
  458. package/numkong/test/README.md +315 -0
  459. package/numkong/test/playwright.config.ts +16 -0
  460. package/numkong/test/swift/Test.swift +534 -0
  461. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  462. package/numkong/test/test-browser-bundle.html +188 -0
  463. package/numkong/test/test-browser-runner.mjs +43 -0
  464. package/numkong/test/test-browser.html +305 -0
  465. package/numkong/test/test-dtypes.mjs +453 -0
  466. package/numkong/test/test-wasm.mjs +267 -0
  467. package/numkong/test/test.cpp +442 -0
  468. package/numkong/test/test.hpp +580 -0
  469. package/numkong/test/test.mjs +363 -0
  470. package/numkong/test/test_base.py +1041 -0
  471. package/numkong/test/test_cast.cpp +242 -0
  472. package/numkong/test/test_cross.hpp +551 -0
  473. package/numkong/test/test_cross.py +467 -0
  474. package/numkong/test/test_cross_amx.cpp +50 -0
  475. package/numkong/test/test_cross_arm.cpp +225 -0
  476. package/numkong/test/test_cross_blas.cpp +354 -0
  477. package/numkong/test/test_cross_loongarch.cpp +90 -0
  478. package/numkong/test/test_cross_power.cpp +73 -0
  479. package/numkong/test/test_cross_rvv.cpp +109 -0
  480. package/numkong/test/test_cross_serial.cpp +181 -0
  481. package/numkong/test/test_cross_sme.cpp +148 -0
  482. package/numkong/test/test_cross_wasm.cpp +141 -0
  483. package/numkong/test/test_cross_x86.cpp +442 -0
  484. package/numkong/test/test_curved.cpp +182 -0
  485. package/numkong/test/test_curved.py +207 -0
  486. package/numkong/test/test_dot.cpp +353 -0
  487. package/numkong/test/test_dot.py +296 -0
  488. package/numkong/test/test_each.cpp +437 -0
  489. package/numkong/test/test_each.py +756 -0
  490. package/numkong/test/test_geospatial.cpp +155 -0
  491. package/numkong/test/test_geospatial.py +241 -0
  492. package/numkong/test/test_maxsim.cpp +138 -0
  493. package/numkong/test/test_maxsim.py +209 -0
  494. package/numkong/test/test_mesh.cpp +203 -0
  495. package/numkong/test/test_mesh.py +394 -0
  496. package/numkong/test/test_probability.cpp +136 -0
  497. package/numkong/test/test_probability.py +166 -0
  498. package/numkong/test/test_reduce.cpp +563 -0
  499. package/numkong/test/test_reduce.py +781 -0
  500. package/numkong/test/test_set.cpp +151 -0
  501. package/numkong/test/test_set.py +120 -0
  502. package/numkong/test/test_similarities.py +579 -0
  503. package/numkong/test/test_sparse.cpp +121 -0
  504. package/numkong/test/test_sparse.py +118 -0
  505. package/numkong/test/test_spatial.cpp +518 -0
  506. package/numkong/test/test_spatial.py +269 -0
  507. package/numkong/test/test_tensor.cpp +884 -0
  508. package/numkong/test/test_tensor.py +1417 -0
  509. package/numkong/test/test_trigonometry.cpp +156 -0
  510. package/numkong/test/test_trigonometry.py +180 -0
  511. package/package.json +7 -8
  512. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  513. package/prebuilds/linux-arm64/usearch.node +0 -0
  514. package/prebuilds/linux-x64/usearch.node +0 -0
  515. package/fp16/CMakeLists.txt +0 -176
  516. package/fp16/LICENSE +0 -11
  517. package/fp16/README.md +0 -20
  518. package/fp16/bench/alt-element.cc +0 -89
  519. package/fp16/bench/from-alt-array.cc +0 -175
  520. package/fp16/bench/from-ieee-array.cc +0 -399
  521. package/fp16/bench/ieee-element.cc +0 -272
  522. package/fp16/bench/to-alt-array.cc +0 -91
  523. package/fp16/bench/to-ieee-array.cc +0 -303
  524. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  525. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  526. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  527. package/fp16/configure.py +0 -65
  528. package/fp16/confu.yaml +0 -8
  529. package/fp16/include/fp16/__init__.py +0 -0
  530. package/fp16/include/fp16/avx.py +0 -50
  531. package/fp16/include/fp16/avx2.py +0 -53
  532. package/fp16/include/fp16/bitcasts.h +0 -92
  533. package/fp16/include/fp16/fp16.h +0 -451
  534. package/fp16/include/fp16/psimd.h +0 -131
  535. package/fp16/include/fp16.h +0 -11
  536. package/fp16/jni/Android.mk +0 -6
  537. package/fp16/jni/Application.mk +0 -4
  538. package/fp16/test/alt-from-fp32-value.cc +0 -498
  539. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  540. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  541. package/fp16/test/alt-to-fp32-value.cc +0 -348
  542. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  543. package/fp16/test/bitcasts.cc +0 -56
  544. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  545. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  546. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  547. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  548. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  549. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  550. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  551. package/fp16/test/peachpy/stubs.py +0 -38
  552. package/fp16/test/tables.cc +0 -530
  553. package/fp16/test/tables.h +0 -13
  554. package/fp16/third-party/THHalf.h +0 -92
  555. package/fp16/third-party/eigen-half.h +0 -170
  556. package/fp16/third-party/float16-compressor.h +0 -80
  557. package/fp16/third-party/half.hpp +0 -2916
  558. package/fp16/third-party/npy-halffloat.h +0 -328
  559. package/simsimd/.git-blame-ignore-revs +0 -2
  560. package/simsimd/CONTRIBUTING.md +0 -321
  561. package/simsimd/Cargo.toml +0 -49
  562. package/simsimd/MANIFEST.in +0 -4
  563. package/simsimd/Package.swift +0 -28
  564. package/simsimd/README.md +0 -1493
  565. package/simsimd/binding.gyp +0 -35
  566. package/simsimd/build.rs +0 -70
  567. package/simsimd/c/lib.c +0 -374
  568. package/simsimd/include/module.modulemap +0 -4
  569. package/simsimd/include/simsimd/binary.h +0 -486
  570. package/simsimd/include/simsimd/curved.h +0 -1541
  571. package/simsimd/include/simsimd/dot.h +0 -1853
  572. package/simsimd/include/simsimd/elementwise.h +0 -2465
  573. package/simsimd/include/simsimd/geospatial.h +0 -43
  574. package/simsimd/include/simsimd/mesh.h +0 -69
  575. package/simsimd/include/simsimd/probability.h +0 -606
  576. package/simsimd/include/simsimd/simsimd.h +0 -2567
  577. package/simsimd/include/simsimd/sparse.h +0 -1384
  578. package/simsimd/include/simsimd/spatial.h +0 -2335
  579. package/simsimd/include/simsimd/types.h +0 -666
  580. package/simsimd/javascript/fallback.ts +0 -233
  581. package/simsimd/javascript/lib.c +0 -125
  582. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  583. package/simsimd/javascript/simsimd.ts +0 -179
  584. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  585. package/simsimd/package-lock.json +0 -1761
  586. package/simsimd/package.json +0 -61
  587. package/simsimd/rust/lib.rs +0 -1821
  588. package/simsimd/scripts/bench.cxx +0 -1170
  589. package/simsimd/scripts/bench.ipynb +0 -187
  590. package/simsimd/scripts/bench.js +0 -127
  591. package/simsimd/scripts/bench.rs +0 -876
  592. package/simsimd/scripts/bench_vectors.py +0 -734
  593. package/simsimd/scripts/bench_vectors_live.py +0 -213
  594. package/simsimd/scripts/test.c +0 -187
  595. package/simsimd/scripts/test.mjs +0 -220
  596. package/simsimd/scripts/test.py +0 -1505
  597. package/simsimd/swift/SimSIMD.swift +0 -149
  598. package/simsimd/swift/Test.swift +0 -128
  599. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  600. /package/{simsimd → numkong}/LICENSE +0 -0
  601. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  602. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -1,1384 +0,0 @@
1
- /**
2
- * @file sparse.h
3
- * @brief SIMD-accelerated functions for Sparse Vectors.
4
- * @author Ash Vardanian
5
- * @date March 21, 2024
6
- *
7
- * Contains:
8
- * - Set Intersection ~ Jaccard Distance
9
- * - Sparse Dot Products, outputting the count and weighted product
10
- *
11
- * For datatypes:
12
- * - u16: for vocabularies under 64 thousand tokens
13
- * - u32: for vocabularies under 4 billion tokens
14
- * - u16 indicies + i16 weights: for weighted word counts
15
- * - u16 indicies + bf16 weights: for sparse matrices
16
- *
17
- * For hardware architectures:
18
- * - x86: Ice Lake, Turin
19
- * - Arm: SVE2
20
- *
21
- * Interestingly, to implement sparse distances and products, the most important function
22
- * is analogous to `std::set_intersection`, that outputs the intersection of two sorted
23
- * sequences. The naive implementation of that function would look like:
24
- *
25
- * std::size_t intersection_size = 0;
26
- * while (i != a_length && j != b_length) {
27
- * scalar_t ai = a[i], bj = b[j];
28
- * intersection_size += ai == bj;
29
- * i += ai < bj;
30
- * j += ai >= bj;
31
- * }
32
- *
33
- * Assuming we are dealing with sparse arrays, most of the time we are just evaluating
34
- * branches and skipping entries. So what if we could skip multiple entries at a time
35
- * searching for the next chunk, where an intersection is possible. For weighted arrays:
36
- *
37
- * double product = 0;
38
- * while (i != a_length && j != b_length) {
39
- * scalar_t ai = a[i], bj = b[j];
40
- * product += ai == bj ? a_weights[i] * b_weights[j] : 0;
41
- * i += ai < bj;
42
- * j += ai >= bj;
43
- * }
44
- *
45
- * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
46
- * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
47
- */
48
- #ifndef SIMSIMD_SPARSE_H
49
- #define SIMSIMD_SPARSE_H
50
-
51
- #include "types.h"
52
-
53
- #ifdef __cplusplus
54
- extern "C" {
55
- #endif
56
-
57
- /* Implements the serial set intersection algorithm, similar to `std::set_intersection in C++ STL`,
58
- * but uses clever galloping logic, if the arrays significantly differ in size.
59
- */
60
- SIMSIMD_PUBLIC void simsimd_intersect_u16_serial( //
61
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
62
- simsimd_size_t a_length, simsimd_size_t b_length, //
63
- simsimd_distance_t *results);
64
- SIMSIMD_PUBLIC void simsimd_intersect_u32_serial( //
65
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
66
- simsimd_size_t a_length, simsimd_size_t b_length, //
67
- simsimd_distance_t *results);
68
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_serial( //
69
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
70
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
71
- simsimd_size_t a_length, simsimd_size_t b_length, //
72
- simsimd_distance_t *results);
73
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_serial( //
74
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
75
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
76
- simsimd_size_t a_length, simsimd_size_t b_length, //
77
- simsimd_distance_t *results);
78
-
79
- /* Implements the most naive set intersection algorithm, similar to `std::set_intersection in C++ STL`,
80
- * naively enumerating the elements of two arrays.
81
- */
82
- SIMSIMD_PUBLIC void simsimd_intersect_u16_accurate( //
83
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
84
- simsimd_size_t a_length, simsimd_size_t b_length, //
85
- simsimd_distance_t *results);
86
- SIMSIMD_PUBLIC void simsimd_intersect_u32_accurate( //
87
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
88
- simsimd_size_t a_length, simsimd_size_t b_length, //
89
- simsimd_distance_t *results);
90
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_accurate( //
91
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
92
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
93
- simsimd_size_t a_length, simsimd_size_t b_length, //
94
- simsimd_distance_t *results);
95
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_accurate( //
96
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
97
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
98
- simsimd_size_t a_length, simsimd_size_t b_length, //
99
- simsimd_distance_t *results);
100
-
101
- /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
102
- * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
103
- */
104
- SIMSIMD_PUBLIC void simsimd_intersect_u16_sve2( //
105
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
106
- simsimd_size_t a_length, simsimd_size_t b_length, //
107
- simsimd_distance_t *results);
108
- SIMSIMD_PUBLIC void simsimd_intersect_u32_sve2( //
109
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
110
- simsimd_size_t a_length, simsimd_size_t b_length, //
111
- simsimd_distance_t *results);
112
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_sve2( //
113
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
114
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
115
- simsimd_size_t a_length, simsimd_size_t b_length, //
116
- simsimd_distance_t *results);
117
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_sve2( //
118
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
119
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
120
- simsimd_size_t a_length, simsimd_size_t b_length, //
121
- simsimd_distance_t *results);
122
-
123
- /* SIMD-powered backends for various generations of AVX512 CPUs.
124
- * Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
125
- * Ice Lake, however, is needed even for the most basic kernels to perform integer matching.
126
- */
127
- SIMSIMD_PUBLIC void simsimd_intersect_u16_ice( //
128
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
129
- simsimd_size_t a_length, simsimd_size_t b_length, //
130
- simsimd_distance_t *results);
131
- SIMSIMD_PUBLIC void simsimd_intersect_u32_ice( //
132
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
133
- simsimd_size_t a_length, simsimd_size_t b_length, //
134
- simsimd_distance_t *results);
135
-
136
- /* SIMD-powered backends for AMD Turin CPUs with cheap VP2INTERSECT instructions.
137
- * On the Intel side, only mobile Tiger Lake support them, but have prohibitively high latency.
138
- */
139
- SIMSIMD_PUBLIC void simsimd_intersect_u16_turin( //
140
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
141
- simsimd_size_t a_length, simsimd_size_t b_length, //
142
- simsimd_distance_t *results);
143
- SIMSIMD_PUBLIC void simsimd_intersect_u32_turin( //
144
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
145
- simsimd_size_t a_length, simsimd_size_t b_length, //
146
- simsimd_distance_t *results);
147
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_turin( //
148
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
149
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
150
- simsimd_size_t a_length, simsimd_size_t b_length, //
151
- simsimd_distance_t *results);
152
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_turin( //
153
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
154
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
155
- simsimd_size_t a_length, simsimd_size_t b_length, //
156
- simsimd_distance_t *results);
157
-
158
- #define SIMSIMD_MAKE_INTERSECT_LINEAR(name, input_type, counter_type) \
159
- SIMSIMD_PUBLIC void simsimd_intersect_##input_type##_##name( \
160
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_size_t a_length, \
161
- simsimd_size_t b_length, simsimd_distance_t *result) { \
162
- simsimd_##counter_type##_t intersection_size = 0; \
163
- simsimd_size_t i = 0, j = 0; \
164
- while (i != a_length && j != b_length) { \
165
- simsimd_##input_type##_t ai = a[i]; \
166
- simsimd_##input_type##_t bj = b[j]; \
167
- intersection_size += ai == bj; \
168
- i += ai < bj; \
169
- j += ai >= bj; \
170
- } \
171
- *result = intersection_size; \
172
- }
173
-
174
- SIMSIMD_MAKE_INTERSECT_LINEAR(accurate, u16, size) // simsimd_intersect_u16_accurate
175
- SIMSIMD_MAKE_INTERSECT_LINEAR(accurate, u32, size) // simsimd_intersect_u32_accurate
176
-
177
- #define SIMSIMD_MAKE_INTERSECT_WEIGHTED(name, variation, input_type, counter_type, weight_type, accumulator_type, \
178
- load_and_convert) \
179
- SIMSIMD_PUBLIC void simsimd_##variation##_##input_type##_##name( \
180
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, \
181
- simsimd_##weight_type##_t const *a_weights, simsimd_##weight_type##_t const *b_weights, \
182
- simsimd_size_t a_length, simsimd_size_t b_length, simsimd_distance_t *results) { \
183
- simsimd_##counter_type##_t intersection_size = 0; \
184
- simsimd_##accumulator_type##_t weights_product = 0; \
185
- simsimd_size_t i = 0, j = 0; \
186
- while (i != a_length && j != b_length) { \
187
- simsimd_##input_type##_t ai = a[i]; \
188
- simsimd_##input_type##_t bj = b[j]; \
189
- int matches = ai == bj; \
190
- simsimd_##accumulator_type##_t awi = load_and_convert(a_weights + i); \
191
- simsimd_##accumulator_type##_t bwi = load_and_convert(b_weights + i); \
192
- weights_product += matches * awi * bwi; \
193
- intersection_size += matches; \
194
- i += ai < bj; \
195
- j += ai >= bj; \
196
- } \
197
- results[0] = intersection_size; \
198
- results[1] = weights_product; \
199
- }
200
-
201
- SIMSIMD_MAKE_INTERSECT_WEIGHTED(accurate, spdot_counts, u16, size, i16, i64,
202
- SIMSIMD_DEREFERENCE) // simsimd_spdot_counts_u16_accurate
203
- SIMSIMD_MAKE_INTERSECT_WEIGHTED(accurate, spdot_weights, u16, size, bf16, f64,
204
- SIMSIMD_BF16_TO_F32) // simsimd_spdot_weights_u16_accurate
205
-
206
- #define SIMSIMD_MAKE_INTERSECT_GALLOPING(name, input_type, counter_type) \
207
- SIMSIMD_PUBLIC simsimd_size_t simsimd_galloping_search_##input_type(simsimd_##input_type##_t const *array, \
208
- simsimd_size_t start, simsimd_size_t length, \
209
- simsimd_##input_type##_t val) { \
210
- simsimd_size_t low = start; \
211
- simsimd_size_t high = start + 1; \
212
- while (high < length && array[high] < val) { \
213
- low = high; \
214
- high = (2 * high < length) ? 2 * high : length; \
215
- } \
216
- while (low < high) { \
217
- simsimd_size_t mid = low + (high - low) / 2; \
218
- if (array[mid] < val) { low = mid + 1; } \
219
- else { high = mid; } \
220
- } \
221
- return low; \
222
- } \
223
- \
224
- SIMSIMD_PUBLIC void simsimd_intersect_##input_type##_##name( \
225
- simsimd_##input_type##_t const *shorter, simsimd_##input_type##_t const *longer, \
226
- simsimd_size_t shorter_length, simsimd_size_t longer_length, simsimd_distance_t *result) { \
227
- /* Swap arrays if necessary, as we want "longer" to be larger than "shorter" */ \
228
- if (longer_length < shorter_length) { \
229
- simsimd_##input_type##_t const *temp = shorter; \
230
- shorter = longer; \
231
- longer = temp; \
232
- simsimd_size_t temp_length = shorter_length; \
233
- shorter_length = longer_length; \
234
- longer_length = temp_length; \
235
- } \
236
- \
237
- /* Use the accurate implementation if galloping is not beneficial */ \
238
- if (longer_length < 64 * shorter_length) { \
239
- simsimd_intersect_##input_type##_accurate(shorter, longer, shorter_length, longer_length, result); \
240
- return; \
241
- } \
242
- \
243
- /* Perform galloping, shrinking the target range */ \
244
- simsimd_##counter_type##_t intersection_size = 0; \
245
- simsimd_size_t j = 0; \
246
- for (simsimd_size_t i = 0; i < shorter_length; ++i) { \
247
- simsimd_##input_type##_t shorter_i = shorter[i]; \
248
- j = simsimd_galloping_search_##input_type(longer, j, longer_length, shorter_i); \
249
- if (j < longer_length && longer[j] == shorter_i) { intersection_size++; } \
250
- } \
251
- *result = intersection_size; \
252
- }
253
-
254
- SIMSIMD_MAKE_INTERSECT_GALLOPING(serial, u16, size) // simsimd_intersect_u16_serial
255
- SIMSIMD_MAKE_INTERSECT_GALLOPING(serial, u32, size) // simsimd_intersect_u32_serial
256
- SIMSIMD_MAKE_INTERSECT_WEIGHTED(serial, spdot_counts, u16, size, i16, i32,
257
- SIMSIMD_DEREFERENCE) // simsimd_spdot_counts_u16_serial
258
- SIMSIMD_MAKE_INTERSECT_WEIGHTED(serial, spdot_weights, u16, size, bf16, f32,
259
- SIMSIMD_BF16_TO_F32) // simsimd_spdot_weights_u16_serial
260
-
261
- /* The AVX-512 implementations are inspired by the "Faster-Than-Native Alternatives
262
- * for x86 VP2INTERSECT Instructions" paper by Guille Diez-Canas, 2022.
263
- *
264
- * https://github.com/mozonaut/vp2intersect
265
- * https://arxiv.org/pdf/2112.06342.pdf
266
- *
267
- * For R&D purposes, it's important to keep the following latencies in mind:
268
- *
269
- * - `_mm512_permutex_epi64` - needs F - 3 cycles latency
270
- * - `_mm512_shuffle_epi8` - needs BW - 1 cycle latency
271
- * - `_mm512_permutexvar_epi16` - needs BW - 4-6 cycles latency
272
- * - `_mm512_permutexvar_epi8` - needs VBMI - 3 cycles latency
273
- */
274
- #if _SIMSIMD_TARGET_X86
275
- #if SIMSIMD_TARGET_ICE
276
- #pragma GCC push_options
277
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "lzcnt", "popcnt", "avx512bw", "avx512vbmi2")
278
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,lzcnt,popcnt,avx512bw,avx512vbmi2"))), \
279
- apply_to = function)
280
-
281
- /**
282
- * @brief Analogous to `_mm512_2intersect_epi16_mask`, but compatible with Ice Lake CPUs,
283
- * slightly faster than the native Tiger Lake implementation, but returns only one mask.
284
- */
285
- SIMSIMD_INTERNAL simsimd_u32_t _simsimd_intersect_u16x32_ice(__m512i a, __m512i b) {
286
- __m512i a1 = _mm512_alignr_epi32(a, a, 4);
287
- __m512i a2 = _mm512_alignr_epi32(a, a, 8);
288
- __m512i a3 = _mm512_alignr_epi32(a, a, 12);
289
-
290
- __m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
291
- __m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
292
- __m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
293
-
294
- __m512i b01 = _mm512_shrdi_epi32(b, b, 16);
295
- __m512i b11 = _mm512_shrdi_epi32(b1, b1, 16);
296
- __m512i b21 = _mm512_shrdi_epi32(b2, b2, 16);
297
- __m512i b31 = _mm512_shrdi_epi32(b3, b3, 16);
298
-
299
- __mmask32 nm00 = _mm512_cmpneq_epi16_mask(a, b);
300
- __mmask32 nm01 = _mm512_cmpneq_epi16_mask(a1, b);
301
- __mmask32 nm02 = _mm512_cmpneq_epi16_mask(a2, b);
302
- __mmask32 nm03 = _mm512_cmpneq_epi16_mask(a3, b);
303
-
304
- __mmask32 nm10 = _mm512_mask_cmpneq_epi16_mask(nm00, a, b01);
305
- __mmask32 nm11 = _mm512_mask_cmpneq_epi16_mask(nm01, a1, b01);
306
- __mmask32 nm12 = _mm512_mask_cmpneq_epi16_mask(nm02, a2, b01);
307
- __mmask32 nm13 = _mm512_mask_cmpneq_epi16_mask(nm03, a3, b01);
308
-
309
- __mmask32 nm20 = _mm512_mask_cmpneq_epi16_mask(nm10, a, b1);
310
- __mmask32 nm21 = _mm512_mask_cmpneq_epi16_mask(nm11, a1, b1);
311
- __mmask32 nm22 = _mm512_mask_cmpneq_epi16_mask(nm12, a2, b1);
312
- __mmask32 nm23 = _mm512_mask_cmpneq_epi16_mask(nm13, a3, b1);
313
-
314
- __mmask32 nm30 = _mm512_mask_cmpneq_epi16_mask(nm20, a, b11);
315
- __mmask32 nm31 = _mm512_mask_cmpneq_epi16_mask(nm21, a1, b11);
316
- __mmask32 nm32 = _mm512_mask_cmpneq_epi16_mask(nm22, a2, b11);
317
- __mmask32 nm33 = _mm512_mask_cmpneq_epi16_mask(nm23, a3, b11);
318
-
319
- __mmask32 nm40 = _mm512_mask_cmpneq_epi16_mask(nm30, a, b2);
320
- __mmask32 nm41 = _mm512_mask_cmpneq_epi16_mask(nm31, a1, b2);
321
- __mmask32 nm42 = _mm512_mask_cmpneq_epi16_mask(nm32, a2, b2);
322
- __mmask32 nm43 = _mm512_mask_cmpneq_epi16_mask(nm33, a3, b2);
323
-
324
- __mmask32 nm50 = _mm512_mask_cmpneq_epi16_mask(nm40, a, b21);
325
- __mmask32 nm51 = _mm512_mask_cmpneq_epi16_mask(nm41, a1, b21);
326
- __mmask32 nm52 = _mm512_mask_cmpneq_epi16_mask(nm42, a2, b21);
327
- __mmask32 nm53 = _mm512_mask_cmpneq_epi16_mask(nm43, a3, b21);
328
-
329
- __mmask32 nm60 = _mm512_mask_cmpneq_epi16_mask(nm50, a, b3);
330
- __mmask32 nm61 = _mm512_mask_cmpneq_epi16_mask(nm51, a1, b3);
331
- __mmask32 nm62 = _mm512_mask_cmpneq_epi16_mask(nm52, a2, b3);
332
- __mmask32 nm63 = _mm512_mask_cmpneq_epi16_mask(nm53, a3, b3);
333
-
334
- __mmask32 nm70 = _mm512_mask_cmpneq_epi16_mask(nm60, a, b31);
335
- __mmask32 nm71 = _mm512_mask_cmpneq_epi16_mask(nm61, a1, b31);
336
- __mmask32 nm72 = _mm512_mask_cmpneq_epi16_mask(nm62, a2, b31);
337
- __mmask32 nm73 = _mm512_mask_cmpneq_epi16_mask(nm63, a3, b31);
338
-
339
- return ~(simsimd_u32_t)(nm70 & simsimd_u32_rol(nm71, 8) & simsimd_u32_rol(nm72, 16) & simsimd_u32_ror(nm73, 8));
340
- }
341
-
342
- /**
343
- * @brief Analogous to `_mm512_2intersect_epi32`, but compatible with Ice Lake CPUs,
344
- * slightly faster than the native Tiger Lake implementation, but returns only one mask.
345
- *
346
- * Some latencies to keep in mind:
347
- *
348
- * - `_mm512_shuffle_epi32` - "VPSHUFD (ZMM, ZMM, I8)":
349
- * - 1 cycle latency on Ice Lake: 1*p5
350
- * - 1 cycle latency on Genoa: 1*FP123
351
- * - `_mm512_mask_cmpneq_epi32_mask` - "VPCMPD (K, ZMM, ZMM, I8)":
352
- * - 3 cycle latency on Ice Lake: 1*p5
353
- * - 1 cycle latency on Genoa: 1*FP01
354
- * - `_mm512_alignr_epi32` - "VPALIGNR (ZMM, ZMM, ZMM, I8)":
355
- * - 1 cycle latency on Ice Lake: 1*p5
356
- * - 2 cycle latency on Genoa: 1*FP12
357
- * - `_mm512_conflict_epi32` - "VPCONFLICTD (ZMM, ZMM)":
358
- * - up to 26 cycles latency on Ice Lake: 11*p0+9*p05+17*p5
359
- * - up to 7 cycle latency on Genoa: 1*FP01+1*FP12
360
- */
361
- SIMSIMD_INTERNAL simsimd_u16_t _simsimd_intersect_u32x16_ice(__m512i a, __m512i b) {
362
- __m512i a1 = _mm512_alignr_epi32(a, a, 4);
363
- __m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
364
- __mmask16 nm00 = _mm512_cmpneq_epi32_mask(a, b);
365
-
366
- __m512i a2 = _mm512_alignr_epi32(a, a, 8);
367
- __m512i a3 = _mm512_alignr_epi32(a, a, 12);
368
- __mmask16 nm01 = _mm512_cmpneq_epi32_mask(a1, b);
369
- __mmask16 nm02 = _mm512_cmpneq_epi32_mask(a2, b);
370
-
371
- __mmask16 nm03 = _mm512_cmpneq_epi32_mask(a3, b);
372
- __mmask16 nm10 = _mm512_mask_cmpneq_epi32_mask(nm00, a, b1);
373
- __mmask16 nm11 = _mm512_mask_cmpneq_epi32_mask(nm01, a1, b1);
374
-
375
- __m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
376
- __mmask16 nm12 = _mm512_mask_cmpneq_epi32_mask(nm02, a2, b1);
377
- __mmask16 nm13 = _mm512_mask_cmpneq_epi32_mask(nm03, a3, b1);
378
- __mmask16 nm20 = _mm512_mask_cmpneq_epi32_mask(nm10, a, b2);
379
-
380
- __m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
381
- __mmask16 nm21 = _mm512_mask_cmpneq_epi32_mask(nm11, a1, b2);
382
- __mmask16 nm22 = _mm512_mask_cmpneq_epi32_mask(nm12, a2, b2);
383
- __mmask16 nm23 = _mm512_mask_cmpneq_epi32_mask(nm13, a3, b2);
384
-
385
- __mmask16 nm0 = _mm512_mask_cmpneq_epi32_mask(nm20, a, b3);
386
- __mmask16 nm1 = _mm512_mask_cmpneq_epi32_mask(nm21, a1, b3);
387
- __mmask16 nm2 = _mm512_mask_cmpneq_epi32_mask(nm22, a2, b3);
388
- __mmask16 nm3 = _mm512_mask_cmpneq_epi32_mask(nm23, a3, b3);
389
-
390
- return ~(simsimd_u16_t)(nm0 & simsimd_u16_rol(nm1, 4) & simsimd_u16_rol(nm2, 8) & simsimd_u16_ror(nm3, 4));
391
- }
392
-
393
- SIMSIMD_PUBLIC void simsimd_intersect_u16_ice( //
394
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
395
- simsimd_size_t a_length, simsimd_size_t b_length, //
396
- simsimd_distance_t *results) {
397
-
398
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
399
- if (a_length < 64 && b_length < 64) {
400
- simsimd_intersect_u16_serial(a, b, a_length, b_length, results);
401
- return;
402
- }
403
-
404
- simsimd_u16_t const *const a_end = a + a_length;
405
- simsimd_u16_t const *const b_end = b + b_length;
406
- simsimd_size_t c = 0;
407
- union vec_t {
408
- __m512i zmm;
409
- simsimd_u16_t u16[32];
410
- simsimd_u8_t u8[64];
411
- } a_vec, b_vec;
412
-
413
- while (a + 32 <= a_end && b + 32 <= b_end) {
414
- a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
415
- b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
416
-
417
- // Intersecting registers with `_simsimd_intersect_u16x32_ice` involves a lot of shuffling
418
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
419
- simsimd_u16_t a_min;
420
- simsimd_u16_t a_max = a_vec.u16[31];
421
- simsimd_u16_t b_min = b_vec.u16[0];
422
- simsimd_u16_t b_max = b_vec.u16[31];
423
-
424
- // If the slices don't overlap, advance the appropriate pointer
425
- while (a_max < b_min && a + 64 <= a_end) {
426
- a += 32;
427
- a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
428
- a_max = a_vec.u16[31];
429
- }
430
- a_min = a_vec.u16[0];
431
- while (b_max < a_min && b + 64 <= b_end) {
432
- b += 32;
433
- b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
434
- b_max = b_vec.u16[31];
435
- }
436
- b_min = b_vec.u16[0];
437
-
438
- __m512i a_last_broadcasted = _mm512_set1_epi16(*(short const *)&a_max);
439
- __m512i b_last_broadcasted = _mm512_set1_epi16(*(short const *)&b_max);
440
- __mmask32 a_step_mask = _mm512_cmple_epu16_mask(a_vec.zmm, b_last_broadcasted);
441
- __mmask32 b_step_mask = _mm512_cmple_epu16_mask(b_vec.zmm, a_last_broadcasted);
442
- a += 32 - _lzcnt_u32((simsimd_u32_t)a_step_mask);
443
- b += 32 - _lzcnt_u32((simsimd_u32_t)b_step_mask);
444
-
445
- // Now we are likely to have some overlap, so we can intersect the registers
446
- __mmask32 a_matches = _simsimd_intersect_u16x32_ice(a_vec.zmm, b_vec.zmm);
447
-
448
- // The paper also contained a very nice procedure for exporting the matches,
449
- // but we don't need it here:
450
- // _mm512_mask_compressstoreu_epi16(c, a_matches, a_vec);
451
- c += _mm_popcnt_u32(a_matches); // MSVC has no `_popcnt32`
452
- }
453
-
454
- simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
455
- *results += c;
456
- }
457
-
458
- SIMSIMD_PUBLIC void simsimd_intersect_u32_ice( //
459
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
460
- simsimd_size_t a_length, simsimd_size_t b_length, //
461
- simsimd_distance_t *results) {
462
-
463
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
464
- if (a_length < 32 && b_length < 32) {
465
- simsimd_intersect_u32_serial(a, b, a_length, b_length, results);
466
- return;
467
- }
468
-
469
- simsimd_u32_t const *const a_end = a + a_length;
470
- simsimd_u32_t const *const b_end = b + b_length;
471
- simsimd_size_t c = 0;
472
- union vec_t {
473
- __m512i zmm;
474
- simsimd_u32_t u32[16];
475
- simsimd_u8_t u8[64];
476
- } a_vec, b_vec;
477
-
478
- while (a + 16 <= a_end && b + 16 <= b_end) {
479
- a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
480
- b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
481
-
482
- // Intersecting registers with `_simsimd_intersect_u32x16_ice` involves a lot of shuffling
483
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
484
- simsimd_u32_t a_min;
485
- simsimd_u32_t a_max = a_vec.u32[15];
486
- simsimd_u32_t b_min = b_vec.u32[0];
487
- simsimd_u32_t b_max = b_vec.u32[15];
488
-
489
- // If the slices don't overlap, advance the appropriate pointer
490
- while (a_max < b_min && a + 32 <= a_end) {
491
- a += 16;
492
- a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
493
- a_max = a_vec.u32[15];
494
- }
495
- a_min = a_vec.u32[0];
496
- while (b_max < a_min && b + 32 <= b_end) {
497
- b += 16;
498
- b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
499
- b_max = b_vec.u32[15];
500
- }
501
- b_min = b_vec.u32[0];
502
-
503
- __m512i a_last_broadcasted = _mm512_set1_epi32(*(int const *)&a_max);
504
- __m512i b_last_broadcasted = _mm512_set1_epi32(*(int const *)&b_max);
505
- __mmask16 a_step_mask = _mm512_cmple_epu32_mask(a_vec.zmm, b_last_broadcasted);
506
- __mmask16 b_step_mask = _mm512_cmple_epu32_mask(b_vec.zmm, a_last_broadcasted);
507
- a += 32 - _lzcnt_u32((simsimd_u32_t)a_step_mask);
508
- b += 32 - _lzcnt_u32((simsimd_u32_t)b_step_mask);
509
-
510
- // Now we are likely to have some overlap, so we can intersect the registers
511
- __mmask16 a_matches = _simsimd_intersect_u32x16_ice(a_vec.zmm, b_vec.zmm);
512
-
513
- // The paper also contained a very nice procedure for exporting the matches,
514
- // but we don't need it here:
515
- // _mm512_mask_compressstoreu_epi32(c, a_matches, a_vec);
516
- c += _mm_popcnt_u32(a_matches); // MSVC has no `_popcnt32`
517
- }
518
-
519
- simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
520
- *results += c;
521
- }
522
-
523
- #pragma clang attribute pop
524
- #pragma GCC pop_options
525
- #endif // SIMSIMD_TARGET_ICE
526
-
527
- #if SIMSIMD_TARGET_TURIN
528
- #pragma GCC push_options
529
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi", "bmi2", "lzcnt", "popcnt", "avx512bw", "avx512vbmi2", \
530
- "avx512bf16", "avx512vnni", "avx512vp2intersect", "avx512dq")
531
- #pragma clang attribute push( \
532
- __attribute__((target( \
533
- "avx2,avx512f,avx512vl,bmi,bmi2,lzcnt,popcnt,avx512bw,avx512vbmi2,avx512bf16,avx512vnni,avx512vp2intersect,avx512dq"))), \
534
- apply_to = function)
535
-
536
- SIMSIMD_PUBLIC void simsimd_intersect_u16_turin( //
537
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
538
- simsimd_size_t a_length, simsimd_size_t b_length, //
539
- simsimd_distance_t *results) {
540
-
541
- //! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
542
- //! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
543
- //! step through 16 entries at a time.
544
- simsimd_u16_t const *const a_end = a + a_length;
545
- simsimd_u16_t const *const b_end = b + b_length;
546
- simsimd_size_t c = 0;
547
- union vec_t {
548
- __m256i ymm;
549
- simsimd_u16_t u16[16];
550
- simsimd_u8_t u8[32];
551
- } a_vec, b_vec;
552
-
553
- // Broadcast index for last element (hoisted outside loop)
554
- __m256i const last_idx = _mm256_set1_epi16(15);
555
- while (a + 16 <= a_end && b + 16 <= b_end) {
556
- a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
557
- b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
558
-
559
- // Intersect the registers
560
- __m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
561
- __m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
562
- __mmask16 a_matches_any_in_b, b_matches_any_in_a;
563
- _mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
564
-
565
- // The paper also contained a very nice procedure for exporting the matches,
566
- // but we don't need it here:
567
- // _mm512_mask_compressstoreu_epi16(c, a_matches_any_in_b, a_vec);
568
- c += _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
569
-
570
- __m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
571
- __m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
572
- __mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
573
- __mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
574
- a += _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
575
- b += _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
576
- }
577
-
578
- simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
579
- *results += c;
580
- }
581
-
582
- SIMSIMD_PUBLIC void simsimd_intersect_u32_turin( //
583
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
584
- simsimd_size_t a_length, simsimd_size_t b_length, //
585
- simsimd_distance_t *results) {
586
-
587
- simsimd_u32_t const *const a_end = a + a_length;
588
- simsimd_u32_t const *const b_end = b + b_length;
589
- simsimd_size_t c = 0;
590
- union vec_t {
591
- __m512i zmm;
592
- simsimd_u32_t u32[16];
593
- simsimd_u8_t u8[64];
594
- } a_vec, b_vec;
595
-
596
- // Broadcast index for last element (hoisted outside loop)
597
- __m512i const last_idx = _mm512_set1_epi32(15);
598
- while (a + 16 <= a_end && b + 16 <= b_end) {
599
- a_vec.zmm = _mm512_loadu_si512((__m512i const *)a);
600
- b_vec.zmm = _mm512_loadu_si512((__m512i const *)b);
601
-
602
- // Intersect the registers
603
- __mmask16 a_matches_any_in_b, b_matches_any_in_a;
604
- _mm512_2intersect_epi32(a_vec.zmm, b_vec.zmm, &a_matches_any_in_b, &b_matches_any_in_a);
605
-
606
- // The paper also contained a very nice procedure for exporting the matches,
607
- // but we don't need it here:
608
- // _mm512_mask_compressstoreu_epi32(c, a_matches_any_in_b, a_vec);
609
- c += _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
610
-
611
- // Pure SIMD broadcasts - no scalar extraction needed
612
- __m512i a_last_broadcasted = _mm512_permutexvar_epi32(last_idx, a_vec.zmm);
613
- __m512i b_last_broadcasted = _mm512_permutexvar_epi32(last_idx, b_vec.zmm);
614
- __mmask16 a_step_mask = _mm512_cmple_epu32_mask(a_vec.zmm, b_last_broadcasted);
615
- __mmask16 b_step_mask = _mm512_cmple_epu32_mask(b_vec.zmm, a_last_broadcasted);
616
- a += _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
617
- b += _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
618
- }
619
-
620
- simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
621
- *results += c;
622
- }
623
-
624
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_turin( //
625
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
626
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
627
- simsimd_size_t a_length, simsimd_size_t b_length, //
628
- simsimd_distance_t *results) {
629
-
630
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
631
- if (a_length < 64 && b_length < 64) {
632
- simsimd_spdot_weights_u16_serial(a, b, a_weights, b_weights, a_length, b_length, results);
633
- return;
634
- }
635
-
636
- //! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
637
- //! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
638
- //! step through 16 entries at a time.
639
- simsimd_u16_t const *const a_end = a + a_length;
640
- simsimd_u16_t const *const b_end = b + b_length;
641
- simsimd_size_t intersection_size = 0;
642
- union vec_t {
643
- __m256i ymm;
644
- __m256 ymmps;
645
- simsimd_u16_t u16[16];
646
- simsimd_u8_t u8[32];
647
- } a_vec, b_vec, product_vec;
648
- product_vec.ymmps = _mm256_setzero_ps();
649
-
650
- // Broadcast index for last element (hoisted outside loop)
651
- __m256i const last_idx = _mm256_set1_epi16(15);
652
- while (a + 16 <= a_end && b + 16 <= b_end) {
653
- a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
654
- b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
655
-
656
- // Intersecting registers with `_mm512_2intersect_epi16_mask` involves a lot of shuffling
657
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
658
- simsimd_u16_t a_min;
659
- simsimd_u16_t a_max = a_vec.u16[15];
660
- simsimd_u16_t b_min = b_vec.u16[0];
661
- simsimd_u16_t b_max = b_vec.u16[15];
662
-
663
- // If the slices don't overlap, advance the appropriate pointer
664
- while (a_max < b_min && a + 32 <= a_end) {
665
- a += 16, a_weights += 16;
666
- a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
667
- a_max = a_vec.u16[15];
668
- }
669
- a_min = a_vec.u16[0];
670
- while (b_max < a_min && b + 32 <= b_end) {
671
- b += 16, b_weights += 16;
672
- b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
673
- b_max = b_vec.u16[15];
674
- }
675
- b_min = b_vec.u16[0];
676
-
677
- // Now we are likely to have some overlap, so we can intersect the registers
678
- __m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
679
- __m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
680
- __mmask16 a_matches_any_in_b, b_matches_any_in_a;
681
- _mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
682
-
683
- // The paper also contained a very nice procedure for exporting the matches,
684
- // but we don't need it here:
685
- // _mm512_mask_compressstoreu_epi16(intersection_size, a_matches_any_in_b, a_vec);
686
- int a_matches_count_in_b = _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
687
- intersection_size += a_matches_count_in_b;
688
-
689
- // Load and shift all the relevant weights to the start of the vector before doing the dot product
690
- if (a_matches_count_in_b) {
691
- __m256i a_weights_vec = _mm256_lddqu_si256((__m256i const *)a_weights);
692
- a_weights_vec = _mm256_maskz_compress_epi16(a_matches_any_in_b, a_weights_vec);
693
- __m256i b_weights_vec = _mm256_lddqu_si256((__m256i const *)b_weights);
694
- b_weights_vec = _mm256_maskz_compress_epi16(b_matches_any_in_a, b_weights_vec);
695
- product_vec.ymmps = _mm256_dpbf16_ps(product_vec.ymmps, (__m256bh)a_weights_vec, (__m256bh)b_weights_vec);
696
- }
697
-
698
- __m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
699
- __m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
700
- __mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
701
- __mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
702
- simsimd_size_t a_step = _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
703
- simsimd_size_t b_step = _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
704
- a += a_step, a_weights += a_step;
705
- b += b_step, b_weights += b_step;
706
- }
707
- simsimd_spdot_weights_u16_serial(a, b, a_weights, b_weights, a_end - a, b_end - b, results);
708
- results[0] += intersection_size;
709
- results[1] += _mm512_reduce_add_ps(_mm512_insertf32x8(_mm512_setzero_ps(), product_vec.ymmps, 0));
710
- }
711
-
712
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_turin( //
713
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
714
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
715
- simsimd_size_t a_length, simsimd_size_t b_length, //
716
- simsimd_distance_t *results) {
717
-
718
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
719
- if (a_length < 64 && b_length < 64) {
720
- simsimd_spdot_counts_u16_serial(a, b, a_weights, b_weights, a_length, b_length, results);
721
- return;
722
- }
723
-
724
- //! There is no such thing as `_mm512_2intersect_epi16`, only the 32-bit variant!
725
- //! So instead of jumping through 32 entries at a time, like on Ice Lake, we will
726
- //! step through 16 entries at a time.
727
- simsimd_u16_t const *const a_end = a + a_length;
728
- simsimd_u16_t const *const b_end = b + b_length;
729
- simsimd_size_t intersection_size = 0;
730
- union vec_t {
731
- __m256i ymm;
732
- simsimd_u16_t u16[16];
733
- simsimd_u8_t u8[32];
734
- } a_vec, b_vec, product_vec;
735
- product_vec.ymm = _mm256_setzero_si256();
736
-
737
- // Broadcast index for last element (hoisted outside loop)
738
- __m256i const last_idx = _mm256_set1_epi16(15);
739
- while (a + 16 <= a_end && b + 16 <= b_end) {
740
- a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
741
- b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
742
-
743
- // Intersecting registers with `_mm512_2intersect_epi16_mask` involves a lot of shuffling
744
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
745
- simsimd_u16_t a_min;
746
- simsimd_u16_t a_max = a_vec.u16[15];
747
- simsimd_u16_t b_min = b_vec.u16[0];
748
- simsimd_u16_t b_max = b_vec.u16[15];
749
-
750
- // If the slices don't overlap, advance the appropriate pointer
751
- while (a_max < b_min && a + 32 <= a_end) {
752
- a += 16, a_weights += 16;
753
- a_vec.ymm = _mm256_lddqu_si256((__m256i const *)a);
754
- a_max = a_vec.u16[15];
755
- }
756
- a_min = a_vec.u16[0];
757
- while (b_max < a_min && b + 32 <= b_end) {
758
- b += 16, b_weights += 16;
759
- b_vec.ymm = _mm256_lddqu_si256((__m256i const *)b);
760
- b_max = b_vec.u16[15];
761
- }
762
- b_min = b_vec.u16[0];
763
-
764
- // Now we are likely to have some overlap, so we can intersect the registers
765
- __m512i a_i32_vec = _mm512_cvtepu16_epi32(a_vec.ymm);
766
- __m512i b_i32_vec = _mm512_cvtepu16_epi32(b_vec.ymm);
767
- __mmask16 a_matches_any_in_b, b_matches_any_in_a;
768
- _mm512_2intersect_epi32(a_i32_vec, b_i32_vec, &a_matches_any_in_b, &b_matches_any_in_a);
769
-
770
- // The paper also contained a very nice procedure for exporting the matches,
771
- // but we don't need it here:
772
- // _mm512_mask_compressstoreu_epi16(intersection_size, a_matches_any_in_b, a_vec);
773
- int a_matches_count_in_b = _mm_popcnt_u32(a_matches_any_in_b); // MSVC has no `_popcnt32`
774
- intersection_size += a_matches_count_in_b;
775
-
776
- // Load and shift all the relevant weights to the start of the vector before doing the dot product
777
- if (a_matches_count_in_b) {
778
- __m256i a_weights_vec = _mm256_lddqu_si256((__m256i const *)a_weights);
779
- a_weights_vec = _mm256_maskz_compress_epi16(a_matches_any_in_b, a_weights_vec);
780
- __m256i b_weights_vec = _mm256_lddqu_si256((__m256i const *)b_weights);
781
- b_weights_vec = _mm256_maskz_compress_epi16(b_matches_any_in_a, b_weights_vec);
782
- product_vec.ymm = _mm256_dpwssds_epi32(product_vec.ymm, a_weights_vec, b_weights_vec);
783
- }
784
-
785
- __m256i a_last_broadcasted = _mm256_permutexvar_epi16(last_idx, a_vec.ymm);
786
- __m256i b_last_broadcasted = _mm256_permutexvar_epi16(last_idx, b_vec.ymm);
787
- __mmask16 a_step_mask = _mm256_cmple_epu16_mask(a_vec.ymm, b_last_broadcasted);
788
- __mmask16 b_step_mask = _mm256_cmple_epu16_mask(b_vec.ymm, a_last_broadcasted);
789
- simsimd_size_t a_step = _tzcnt_u32(~(simsimd_u32_t)a_step_mask | 0x10000);
790
- simsimd_size_t b_step = _tzcnt_u32(~(simsimd_u32_t)b_step_mask | 0x10000);
791
- a += a_step, a_weights += a_step;
792
- b += b_step, b_weights += b_step;
793
- }
794
-
795
- simsimd_spdot_counts_u16_serial(a, b, a_weights, b_weights, a_end - a, b_end - b, results);
796
- results[0] += intersection_size;
797
- results[1] += _mm512_reduce_add_epi32(_mm512_inserti64x4(_mm512_setzero_si512(), product_vec.ymm, 0));
798
- }
799
-
800
- #pragma clang attribute pop
801
- #pragma GCC pop_options
802
- #endif // SIMSIMD_TARGET_TURIN
803
- #endif // _SIMSIMD_TARGET_X86
804
-
805
- #if _SIMSIMD_TARGET_ARM
806
- #if SIMSIMD_TARGET_NEON
807
- #pragma GCC push_options
808
- #pragma GCC target("arch=armv8-a")
809
- #pragma clang attribute push(__attribute__((target("arch=armv8-a"))), apply_to = function)
810
-
811
- /**
812
- * @brief Uses `vshrn` to produce a bitmask, similar to `movemask` in SSE.
813
- * https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
814
- */
815
- SIMSIMD_INTERNAL simsimd_u64_t _simsimd_u8_to_u4_neon(uint8x16_t vec) {
816
- return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vec), 4)), 0);
817
- }
818
-
819
- SIMSIMD_INTERNAL int _simsimd_clz_u64(simsimd_u64_t x) {
820
- // On GCC and Clang use the builtin, otherwise use the generic implementation
821
- #if defined(__GNUC__) || defined(__clang__)
822
- return __builtin_clzll(x);
823
- #else
824
- int n = 0;
825
- while ((x & 0x8000000000000000ull) == 0) n++, x <<= 1;
826
- return n;
827
- #endif
828
- }
829
-
830
- SIMSIMD_INTERNAL uint32x4_t _simsimd_intersect_u32x4_neon(uint32x4_t a, uint32x4_t b) {
831
- uint32x4_t b1 = vextq_u32(b, b, 1);
832
- uint32x4_t b2 = vextq_u32(b, b, 2);
833
- uint32x4_t b3 = vextq_u32(b, b, 3);
834
- uint32x4_t nm00 = vceqq_u32(a, b);
835
- uint32x4_t nm01 = vceqq_u32(a, b1);
836
- uint32x4_t nm02 = vceqq_u32(a, b2);
837
- uint32x4_t nm03 = vceqq_u32(a, b3);
838
- uint32x4_t nm = vorrq_u32(vorrq_u32(nm00, nm01), vorrq_u32(nm02, nm03));
839
- return nm;
840
- }
841
-
842
- SIMSIMD_INTERNAL uint16x8_t _simsimd_intersect_u16x8_neon(uint16x8_t a, uint16x8_t b) {
843
- uint16x8_t b1 = vextq_u16(b, b, 1);
844
- uint16x8_t b2 = vextq_u16(b, b, 2);
845
- uint16x8_t b3 = vextq_u16(b, b, 3);
846
- uint16x8_t b4 = vextq_u16(b, b, 4);
847
- uint16x8_t b5 = vextq_u16(b, b, 5);
848
- uint16x8_t b6 = vextq_u16(b, b, 6);
849
- uint16x8_t b7 = vextq_u16(b, b, 7);
850
- uint16x8_t nm00 = vceqq_u16(a, b);
851
- uint16x8_t nm01 = vceqq_u16(a, b1);
852
- uint16x8_t nm02 = vceqq_u16(a, b2);
853
- uint16x8_t nm03 = vceqq_u16(a, b3);
854
- uint16x8_t nm04 = vceqq_u16(a, b4);
855
- uint16x8_t nm05 = vceqq_u16(a, b5);
856
- uint16x8_t nm06 = vceqq_u16(a, b6);
857
- uint16x8_t nm07 = vceqq_u16(a, b7);
858
- uint16x8_t nm = vorrq_u16(vorrq_u16(vorrq_u16(nm00, nm01), vorrq_u16(nm02, nm03)),
859
- vorrq_u16(vorrq_u16(nm04, nm05), vorrq_u16(nm06, nm07)));
860
- return nm;
861
- }
862
-
863
- SIMSIMD_PUBLIC void simsimd_intersect_u16_neon( //
864
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
865
- simsimd_size_t a_length, simsimd_size_t b_length, //
866
- simsimd_distance_t *results) {
867
-
868
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
869
- if (a_length < 32 && b_length < 32) {
870
- simsimd_intersect_u16_serial(a, b, a_length, b_length, results);
871
- return;
872
- }
873
-
874
- simsimd_u16_t const *const a_end = a + a_length;
875
- simsimd_u16_t const *const b_end = b + b_length;
876
- union vec_t {
877
- uint16x8_t u16x8;
878
- simsimd_u16_t u16[8];
879
- simsimd_u8_t u8[16];
880
- } a_vec, b_vec, c_counts_vec;
881
- c_counts_vec.u16x8 = vdupq_n_u16(0);
882
-
883
- while (a + 8 <= a_end && b + 8 <= b_end) {
884
- a_vec.u16x8 = vld1q_u16(a);
885
- b_vec.u16x8 = vld1q_u16(b);
886
-
887
- // Intersecting registers with `_simsimd_intersect_u16x8_neon` involves a lot of shuffling
888
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
889
- simsimd_u16_t a_min;
890
- simsimd_u16_t a_max = a_vec.u16[7];
891
- simsimd_u16_t b_min = b_vec.u16[0];
892
- simsimd_u16_t b_max = b_vec.u16[7];
893
-
894
- // If the slices don't overlap, advance the appropriate pointer
895
- while (a_max < b_min && a + 16 <= a_end) {
896
- a += 8;
897
- a_vec.u16x8 = vld1q_u16(a);
898
- a_max = a_vec.u16[7];
899
- }
900
- a_min = a_vec.u16[0];
901
- while (b_max < a_min && b + 16 <= b_end) {
902
- b += 8;
903
- b_vec.u16x8 = vld1q_u16(b);
904
- b_max = b_vec.u16[7];
905
- }
906
- b_min = b_vec.u16[0];
907
-
908
- // Now we are likely to have some overlap, so we can intersect the registers.
909
- // We can do it by performing a population count at every cycle, but it's not the cheapest in terms of cycles.
910
- //
911
- // simsimd_u64_t a_matches = __builtin_popcountll(
912
- // _simsimd_u8_to_u4_neon(vreinterpretq_u8_u16(
913
- // _simsimd_intersect_u16x8_neon(a_vec.u16x8, b_vec.u16x8))));
914
- // c += a_matches / 8;
915
- //
916
- // Alternatively, we can we can transform match-masks into "ones", accumulate them between the cycles,
917
- // and merge all together in the end.
918
- uint16x8_t a_matches = _simsimd_intersect_u16x8_neon(a_vec.u16x8, b_vec.u16x8);
919
- c_counts_vec.u16x8 = vaddq_u16(c_counts_vec.u16x8, vandq_u16(a_matches, vdupq_n_u16(1)));
920
-
921
- // Counting leading zeros is tricky. On Arm we can use inline Assembly to get the result,
922
- // but MSVC doesn't support that:
923
- //
924
- // SIMSIMD_INTERNAL int _simsimd_clz_u64(simsimd_u64_t value) {
925
- // simsimd_u64_t result;
926
- // __asm__("clz %x0, %x1" : "=r"(result) : "r"(value));
927
- // return (int)result;
928
- // }
929
- //
930
- // Alternatively, we can use the `vclz_u32` NEON intrinsic.
931
- // It will compute the leading zeros number for both `a_step` and `b_step` in parallel.
932
- uint16x8_t a_last_broadcasted = vdupq_n_u16(a_max);
933
- uint16x8_t b_last_broadcasted = vdupq_n_u16(b_max);
934
- simsimd_u64_t a_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
935
- vreinterpretq_u8_u16(vcleq_u16(a_vec.u16x8, b_last_broadcasted))));
936
- simsimd_u64_t b_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
937
- vreinterpretq_u8_u16(vcleq_u16(b_vec.u16x8, a_last_broadcasted))));
938
- a += (64 - a_step) / 8;
939
- b += (64 - b_step) / 8;
940
- }
941
-
942
- simsimd_intersect_u16_serial(a, b, a_end - a, b_end - b, results);
943
- *results += vaddvq_u16(c_counts_vec.u16x8);
944
- }
945
-
946
- SIMSIMD_PUBLIC void simsimd_intersect_u32_neon( //
947
- simsimd_u32_t const *a, simsimd_u32_t const *b, //
948
- simsimd_size_t a_length, simsimd_size_t b_length, //
949
- simsimd_distance_t *results) {
950
-
951
- // The baseline implementation for very small arrays (2 registers or less) can be quite simple:
952
- if (a_length < 32 && b_length < 32) {
953
- simsimd_intersect_u32_serial(a, b, a_length, b_length, results);
954
- return;
955
- }
956
-
957
- simsimd_u32_t const *const a_end = a + a_length;
958
- simsimd_u32_t const *const b_end = b + b_length;
959
- union vec_t {
960
- uint32x4_t u32x4;
961
- simsimd_u32_t u32[4];
962
- simsimd_u8_t u8[16];
963
- } a_vec, b_vec, c_counts_vec;
964
- c_counts_vec.u32x4 = vdupq_n_u32(0);
965
-
966
- while (a + 4 <= a_end && b + 4 <= b_end) {
967
- a_vec.u32x4 = vld1q_u32(a);
968
- b_vec.u32x4 = vld1q_u32(b);
969
-
970
- // Intersecting registers with `_simsimd_intersect_u32x4_neon` involves a lot of shuffling
971
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
972
- simsimd_u32_t a_min;
973
- simsimd_u32_t a_max = a_vec.u32[3];
974
- simsimd_u32_t b_min = b_vec.u32[0];
975
- simsimd_u32_t b_max = b_vec.u32[3];
976
-
977
- // If the slices don't overlap, advance the appropriate pointer
978
- while (a_max < b_min && a + 8 <= a_end) {
979
- a += 4;
980
- a_vec.u32x4 = vld1q_u32(a);
981
- a_max = a_vec.u32[3];
982
- }
983
- a_min = a_vec.u32[0];
984
- while (b_max < a_min && b + 8 <= b_end) {
985
- b += 4;
986
- b_vec.u32x4 = vld1q_u32(b);
987
- b_max = b_vec.u32[3];
988
- }
989
- b_min = b_vec.u32[0];
990
-
991
- // Now we are likely to have some overlap, so we can intersect the registers
992
- // We can do it by performing a population count at every cycle, but it's not the cheapest in terms of cycles.
993
- //
994
- // simsimd_u64_t a_matches = __builtin_popcountll(
995
- // _simsimd_u8_to_u4_neon(vreinterpretq_u8_u32(
996
- // _simsimd_intersect_u32x4_neon(a_vec.u32x4, b_vec.u32x4))));
997
- // c += a_matches / 16;
998
- //
999
- // Alternatively, we can we can transform match-masks into "ones", accumulate them between the cycles,
1000
- // and merge all together in the end.
1001
- uint32x4_t a_matches = _simsimd_intersect_u32x4_neon(a_vec.u32x4, b_vec.u32x4);
1002
- c_counts_vec.u32x4 = vaddq_u32(c_counts_vec.u32x4, vandq_u32(a_matches, vdupq_n_u32(1)));
1003
-
1004
- uint32x4_t a_last_broadcasted = vdupq_n_u32(a_max);
1005
- uint32x4_t b_last_broadcasted = vdupq_n_u32(b_max);
1006
- simsimd_u64_t a_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
1007
- vreinterpretq_u8_u32(vcleq_u32(a_vec.u32x4, b_last_broadcasted))));
1008
- simsimd_u64_t b_step = _simsimd_clz_u64(_simsimd_u8_to_u4_neon( //
1009
- vreinterpretq_u8_u32(vcleq_u32(b_vec.u32x4, a_last_broadcasted))));
1010
- a += (64 - a_step) / 16;
1011
- b += (64 - b_step) / 16;
1012
- }
1013
-
1014
- simsimd_intersect_u32_serial(a, b, a_end - a, b_end - b, results);
1015
- *results += vaddvq_u32(c_counts_vec.u32x4);
1016
- }
1017
-
1018
- #pragma clang attribute pop
1019
- #pragma GCC pop_options
1020
- #endif // SIMSIMD_TARGET_NEON
1021
-
1022
- /* SVE2 introduces many new integer-oriented instructions, extending some of the NEON functionality
1023
- * to variable-length SVE registers. Those include "compare multiple" intrinsics:
1024
- *
1025
- * - `svmatch[_u16]` that matches each scalar in first vector against all members of a 128-bit lane in the second.
1026
- * - `svhistcnt[_s32]_z` does something similar, performing an inclusive prefix scan.
1027
- * - `svtbx[_u16]` does extended table lookup
1028
- *
1029
- * Other notable instructions:
1030
- *
1031
- * - `DUP`: Broadcast indexed predicate element
1032
- * https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/DUP--predicate---Broadcast-indexed-predicate-element-?lang=en
1033
- * - `SCLAMP` and `UCLAMP`: clamp values, i.e. combined min+max
1034
- * https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/SCLAMP--Signed-clamp-to-minimum-maximum-vector-?lang=en
1035
- * https://developer.arm.com/documentation/ddi0602/2021-06/SVE-Instructions/UCLAMP--Unsigned-clamp-to-minimum-maximum-vector-?lang=en
1036
- * - `TBLQ`: Table lookup quadword
1037
- * https://developer.arm.com/documentation/ddi0602/2022-12/SVE-Instructions/TBLQ--Programmable-table-lookup-within-each-quadword-vector-segment--zeroing--?lang=en
1038
- *
1039
- * Great resources for SVE2 intrinsics:
1040
- *
1041
- * > ARM’s Scalable Vector Extensions: A Critical Look at SVE2 For Integer Workloads
1042
- * https://gist.github.com/zingaburga/805669eb891c820bd220418ee3f0d6bd
1043
- */
1044
- #if SIMSIMD_TARGET_SVE2
1045
- #pragma GCC push_options
1046
- #pragma GCC target("arch=armv8.2-a+sve+sve2")
1047
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+sve2"))), apply_to = function)
1048
-
1049
- SIMSIMD_PUBLIC void simsimd_intersect_u16_sve2( //
1050
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
1051
- simsimd_size_t a_length,
1052
- simsimd_size_t b_length, //
1053
- simsimd_distance_t *results) {
1054
-
1055
- // A single SVE lane is 128 bits wide, so one lane fits 8 values.
1056
- simsimd_size_t const register_size = svcnth();
1057
- simsimd_size_t const lanes_count = register_size / 8;
1058
- simsimd_size_t a_idx = 0, b_idx = 0;
1059
- simsimd_size_t c = 0;
1060
-
1061
- while (a_idx < a_length && b_idx < b_length) {
1062
- // Load `a_member` and broadcast it, load `b_members_vec` from memory
1063
- svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
1064
- svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
1065
- svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
1066
- svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
1067
-
1068
- // Intersecting registers with `svmatch_u16` involves a lot of shuffling
1069
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
1070
- simsimd_u16_t a_min;
1071
- simsimd_u16_t a_max = svlastb(a_progress, a_vec);
1072
- simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
1073
- simsimd_u16_t b_max = svlastb(b_progress, b_vec);
1074
-
1075
- // If the slices don't overlap, advance the appropriate pointer
1076
- while (a_max < b_min && (a_idx + register_size) <= a_length) {
1077
- a_idx += register_size;
1078
- a_progress = svwhilelt_b16_u64(a_idx, a_length);
1079
- a_vec = svld1_u16(a_progress, a + a_idx);
1080
- a_max = svlastb(a_progress, a_vec);
1081
- }
1082
- a_min = svlasta(svpfalse_b(), a_vec);
1083
- while (b_max < a_min && (b_idx + register_size) <= b_length) {
1084
- b_idx += register_size;
1085
- b_progress = svwhilelt_b16_u64(b_idx, b_length);
1086
- b_vec = svld1_u16(b_progress, b + b_idx);
1087
- b_max = svlastb(b_progress, b_vec);
1088
- }
1089
- b_min = svlasta(svpfalse_b(), b_vec);
1090
-
1091
- // Before we evaluate the intersection size, obfurscating the order in `b_vec`,
1092
- // let's estimate how much we will need to advance the pointers afterwards.
1093
- // For that, we don't even need to broadcast the values in SVE, as the whole
1094
- // register can be compared against a scalar:
1095
- //
1096
- // svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
1097
- // svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
1098
- svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
1099
- svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
1100
- simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
1101
- simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
1102
-
1103
- // Compare `a_vec` with each lane of `b_vec`
1104
- svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
1105
- for (simsimd_size_t i = 1; i < lanes_count; i++) {
1106
- b_vec = svext_u16(b_vec, b_vec, 8);
1107
- equal_mask = svorr_z(svptrue_b16(), equal_mask, svmatch_u16(a_progress, a_vec, b_vec));
1108
- }
1109
- simsimd_size_t equal_count = svcntp_b16(svptrue_b16(), equal_mask);
1110
-
1111
- // Advance
1112
- a_idx += a_step;
1113
- b_idx += b_step;
1114
- c += equal_count;
1115
- }
1116
- *results = c;
1117
- }
1118
-
1119
- SIMSIMD_PUBLIC void simsimd_intersect_u32_sve2(simsimd_u32_t const *a, simsimd_u32_t const *b, simsimd_size_t a_length,
1120
- simsimd_size_t b_length, simsimd_distance_t *results) {
1121
-
1122
- // A single SVE lane is 128 bits wide, so one lane fits 4 values.
1123
- simsimd_size_t const register_size = svcntw();
1124
- simsimd_size_t const lanes_count = register_size / 4;
1125
- simsimd_size_t a_idx = 0, b_idx = 0;
1126
- simsimd_size_t c = 0;
1127
-
1128
- while (a_idx < a_length && b_idx < b_length) {
1129
- // Load `a_member` and broadcast it, load `b_members_vec` from memory
1130
- svbool_t a_progress = svwhilelt_b32_u64(a_idx, a_length);
1131
- svbool_t b_progress = svwhilelt_b32_u64(b_idx, b_length);
1132
- svuint32_t a_vec = svld1_u32(a_progress, a + a_idx);
1133
- svuint32_t b_vec = svld1_u32(b_progress, b + b_idx);
1134
-
1135
- // Intersecting registers with `svmatch_u16` involves a lot of shuffling
1136
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
1137
- simsimd_u32_t a_min;
1138
- simsimd_u32_t a_max = svlastb(a_progress, a_vec);
1139
- simsimd_u32_t b_min = svlasta(svpfalse_b(), b_vec);
1140
- simsimd_u32_t b_max = svlastb(b_progress, b_vec);
1141
-
1142
- // If the slices don't overlap, advance the appropriate pointer
1143
- while (a_max < b_min && (a_idx + register_size) <= a_length) {
1144
- a_idx += register_size;
1145
- a_progress = svwhilelt_b32_u64(a_idx, a_length);
1146
- a_vec = svld1_u32(a_progress, a + a_idx);
1147
- a_max = svlastb(a_progress, a_vec);
1148
- }
1149
- a_min = svlasta(svpfalse_b(), a_vec);
1150
- while (b_max < a_min && (b_idx + register_size) <= b_length) {
1151
- b_idx += register_size;
1152
- b_progress = svwhilelt_b32_u64(b_idx, b_length);
1153
- b_vec = svld1_u32(b_progress, b + b_idx);
1154
- b_max = svlastb(b_progress, b_vec);
1155
- }
1156
- b_min = svlasta(svpfalse_b(), b_vec);
1157
-
1158
- // Before we evaluate the intersection size, obfurscating the order in `b_vec`,
1159
- // let's estimate how much we will need to advance the pointers afterwards.
1160
- // For that, we don't even need to broadcast the values in SVE, as the whole
1161
- // register can be compared against a scalar:
1162
- //
1163
- // svuint32_t a_last_broadcasted = svdup_n_u32(a_max);
1164
- // svuint32_t b_last_broadcasted = svdup_n_u32(b_max);
1165
- svbool_t a_mask = svcmple_n_u32(a_progress, a_vec, b_max);
1166
- svbool_t b_mask = svcmple_n_u32(b_progress, b_vec, a_max);
1167
- simsimd_u64_t a_step = svcntp_b32(a_progress, a_mask);
1168
- simsimd_u64_t b_step = svcntp_b32(b_progress, b_mask);
1169
-
1170
- // Comparing `a_vec` with each lane of `b_vec` can't be done with `svmatch`,
1171
- // the same way as in `simsimd_intersect_u16_sve2`, as that instruction is only
1172
- // available for 8-bit and 16-bit integers.
1173
- //
1174
- // svbool_t equal_mask = svpfalse_b();
1175
- // for (simsimd_size_t i = 0; i < register_size; i++) {
1176
- // equal_mask = svorr_z(svptrue_b32(), equal_mask, svcmpeq_u32(a_progress, a_vec, b_vec));
1177
- // b_vec = svext_u32(b_vec, b_vec, 1);
1178
- // }
1179
- // simsimd_size_t equal_count = svcntp_b32(a_progress, equal_mask);
1180
- //
1181
- // Alternatively, one can use histogram instructions, like `svhistcnt_u32_z`.
1182
- // They practically compute the prefix-matching count, which is equivalent to
1183
- // the lower triangle of the row-major intersection matrix.
1184
- // To compute the upper triangle, we can reverse (with `svrev_b32`) the order of
1185
- // elements and repeat the operation, accumulating the results for top and bottom.
1186
- // Let's look at 4x element registers as an example:
1187
- //
1188
- // ⊐ α = {A, B, C, D}, β = {X, Y, Z, W}:
1189
- //
1190
- // hist(α, β): hist(α_rev, β_rev):
1191
- //
1192
- // X Y Z W W Z Y X
1193
- // A 1 0 0 0 D 1 0 0 0
1194
- // B 1 1 0 0 C 1 1 0 0
1195
- // C 1 1 1 0 B 1 1 1 0
1196
- // D 1 1 1 1 A 1 1 1 1
1197
- //
1198
- svuint32_t hist_lower = svhistcnt_u32_z(a_progress, a_vec, b_vec);
1199
- svuint32_t a_rev_vec = svrev_u32(a_vec);
1200
- svuint32_t b_rev_vec = svrev_u32(b_vec);
1201
- svuint32_t hist_upper = svrev_u32(svhistcnt_u32_z(svptrue_b32(), a_rev_vec, b_rev_vec));
1202
- svuint32_t hist = svorr_u32_x(a_progress, hist_lower, hist_upper);
1203
- svbool_t equal_mask = svcmpne_n_u32(a_progress, hist, 0);
1204
- simsimd_size_t equal_count = svcntp_b32(a_progress, equal_mask);
1205
-
1206
- // Advance
1207
- a_idx += a_step;
1208
- b_idx += b_step;
1209
- c += equal_count;
1210
- }
1211
- *results = c;
1212
- }
1213
-
1214
- SIMSIMD_PUBLIC void simsimd_spdot_counts_u16_sve2( //
1215
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
1216
- simsimd_i16_t const *a_weights, simsimd_i16_t const *b_weights, //
1217
- simsimd_size_t a_length, simsimd_size_t b_length, //
1218
- simsimd_distance_t *results) {
1219
-
1220
- // A single SVE lane is 128 bits wide, so one lane fits 8 values.
1221
- simsimd_size_t const register_size = svcnth();
1222
- simsimd_size_t const lanes_count = register_size / 8;
1223
- simsimd_size_t a_idx = 0, b_idx = 0;
1224
- svint64_t product_vec = svdupq_n_s64(0, 0);
1225
- simsimd_size_t intersection_size = 0;
1226
-
1227
- while (a_idx < a_length && b_idx < b_length) {
1228
- // Load `a_member` and broadcast it, load `b_members_vec` from memory
1229
- svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
1230
- svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
1231
- svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
1232
- svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
1233
-
1234
- // Intersecting registers with `svmatch_u16` involves a lot of shuffling
1235
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
1236
- simsimd_u16_t a_min;
1237
- simsimd_u16_t a_max = svlastb(a_progress, a_vec);
1238
- simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
1239
- simsimd_u16_t b_max = svlastb(b_progress, b_vec);
1240
-
1241
- // If the slices don't overlap, advance the appropriate pointer
1242
- while (a_max < b_min && (a_idx + register_size) <= a_length) {
1243
- a_idx += register_size;
1244
- a_progress = svwhilelt_b16_u64(a_idx, a_length);
1245
- a_vec = svld1_u16(a_progress, a + a_idx);
1246
- a_max = svlastb(a_progress, a_vec);
1247
- }
1248
- a_min = svlasta(svpfalse_b(), a_vec);
1249
- while (b_max < a_min && (b_idx + register_size) <= b_length) {
1250
- b_idx += register_size;
1251
- b_progress = svwhilelt_b16_u64(b_idx, b_length);
1252
- b_vec = svld1_u16(b_progress, b + b_idx);
1253
- b_max = svlastb(b_progress, b_vec);
1254
- }
1255
- b_min = svlasta(svpfalse_b(), b_vec);
1256
-
1257
- // Before we evaluate the intersection size, obfurscating the order in `b_vec`,
1258
- // let's estimate how much we will need to advance the pointers afterwards.
1259
- // For that, we don't even need to broadcast the values in SVE, as the whole
1260
- // register can be compared against a scalar:
1261
- //
1262
- // svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
1263
- // svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
1264
- svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
1265
- svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
1266
- simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
1267
- simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
1268
-
1269
- // Compare `a_vec` with each lane of `b_vec`
1270
- svint16_t a_weights_vec = svld1_s16(a_progress, a_weights + a_idx);
1271
- svint16_t b_weights_vec = svld1_s16(b_progress, b_weights + b_idx);
1272
- for (simsimd_size_t i = 0; i < lanes_count; i++) {
1273
- svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
1274
- svint16_t b_equal_weights_vec = svsel_s16(equal_mask, b_weights_vec, svdup_n_s16(0.f));
1275
- product_vec = svdot_s64(product_vec, a_weights_vec, b_equal_weights_vec);
1276
- b_vec = svext_u16(b_vec, b_vec, 8);
1277
- intersection_size += svcntp_b16(svptrue_b16(), equal_mask);
1278
- }
1279
-
1280
- // Advance
1281
- a_idx += a_step;
1282
- b_idx += b_step;
1283
- }
1284
- results[0] = (simsimd_distance_t)intersection_size;
1285
- results[1] = svaddv_s64(svptrue_b64(), product_vec);
1286
- }
1287
-
1288
- #pragma clang attribute pop
1289
- #pragma GCC pop_options
1290
- #endif // SIMSIMD_TARGET_SVE2
1291
-
1292
- #if SIMSIMD_TARGET_SVE2 && SIMSIMD_TARGET_SVE_BF16
1293
- #pragma GCC push_options
1294
- #pragma GCC target("arch=armv8.6-a+sve+sve2+bf16")
1295
- #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+sve+sve2+bf16"))), apply_to = function)
1296
-
1297
- SIMSIMD_PUBLIC void simsimd_spdot_weights_u16_sve2( //
1298
- simsimd_u16_t const *a, simsimd_u16_t const *b, //
1299
- simsimd_bf16_t const *a_weights, simsimd_bf16_t const *b_weights, //
1300
- simsimd_size_t a_length, simsimd_size_t b_length, //
1301
- simsimd_distance_t *results) {
1302
-
1303
- // A single SVE lane is 128 bits wide, so one lane fits 8 values.
1304
- simsimd_size_t const register_size = svcnth();
1305
- simsimd_size_t const lanes_count = register_size / 8;
1306
- simsimd_size_t a_idx = 0, b_idx = 0;
1307
- svfloat32_t product_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
1308
- simsimd_size_t intersection_size = 0;
1309
-
1310
- while (a_idx < a_length && b_idx < b_length) {
1311
- // Load `a_member` and broadcast it, load `b_members_vec` from memory
1312
- svbool_t a_progress = svwhilelt_b16_u64(a_idx, a_length);
1313
- svbool_t b_progress = svwhilelt_b16_u64(b_idx, b_length);
1314
- svuint16_t a_vec = svld1_u16(a_progress, a + a_idx);
1315
- svuint16_t b_vec = svld1_u16(b_progress, b + b_idx);
1316
-
1317
- // Intersecting registers with `svmatch_u16` involves a lot of shuffling
1318
- // and comparisons, so we want to avoid it if the slices don't overlap at all..
1319
- simsimd_u16_t a_min;
1320
- simsimd_u16_t a_max = svlastb(a_progress, a_vec);
1321
- simsimd_u16_t b_min = svlasta(svpfalse_b(), b_vec);
1322
- simsimd_u16_t b_max = svlastb(b_progress, b_vec);
1323
-
1324
- // If the slices don't overlap, advance the appropriate pointer
1325
- while (a_max < b_min && (a_idx + register_size) <= a_length) {
1326
- a_idx += register_size;
1327
- a_progress = svwhilelt_b16_u64(a_idx, a_length);
1328
- a_vec = svld1_u16(a_progress, a + a_idx);
1329
- a_max = svlastb(a_progress, a_vec);
1330
- }
1331
- a_min = svlasta(svpfalse_b(), a_vec);
1332
- while (b_max < a_min && (b_idx + register_size) <= b_length) {
1333
- b_idx += register_size;
1334
- b_progress = svwhilelt_b16_u64(b_idx, b_length);
1335
- b_vec = svld1_u16(b_progress, b + b_idx);
1336
- b_max = svlastb(b_progress, b_vec);
1337
- }
1338
- b_min = svlasta(svpfalse_b(), b_vec);
1339
-
1340
- // Before we evaluate the intersection size, obfurscating the order in `b_vec`,
1341
- // let's estimate how much we will need to advance the pointers afterwards.
1342
- // For that, we don't even need to broadcast the values in SVE, as the whole
1343
- // register can be compared against a scalar:
1344
- //
1345
- // svuint16_t a_last_broadcasted = svdup_n_u16(a_max);
1346
- // svuint16_t b_last_broadcasted = svdup_n_u16(b_max);
1347
- svbool_t a_mask = svcmple_n_u16(a_progress, a_vec, b_max);
1348
- svbool_t b_mask = svcmple_n_u16(b_progress, b_vec, a_max);
1349
- simsimd_u64_t a_step = svcntp_b16(a_progress, a_mask);
1350
- simsimd_u64_t b_step = svcntp_b16(b_progress, b_mask);
1351
-
1352
- // Compare `a_vec` with each lane of `b_vec`
1353
- svbfloat16_t a_weights_vec = svld1_bf16(a_progress, (__bf16 const *)a_weights + a_idx);
1354
- svbfloat16_t b_weights_vec = svld1_bf16(b_progress, (__bf16 const *)b_weights + b_idx);
1355
- for (simsimd_size_t i = 0; i < lanes_count; i++) {
1356
- svbool_t equal_mask = svmatch_u16(a_progress, a_vec, b_vec);
1357
- //! The `svsel_bf16` intrinsic is broken in many compilers, not returning the correct type.
1358
- //! So we reinterprete floats as integers and apply `svsel_s16`, but the `svreinterpret_s16_bs16`
1359
- //! and `svreinterpret_bf16_s16` are not always properly defined!
1360
- svint16_t b_equal_weights_vec =
1361
- svsel_s16(equal_mask, svreinterpret_s16_bf16(b_weights_vec), svdup_n_s16(0));
1362
- product_vec = svbfdot_f32(product_vec, a_weights_vec, svreinterpret_bf16_s16(b_equal_weights_vec));
1363
- b_vec = svext_u16(b_vec, b_vec, 8);
1364
- intersection_size += svcntp_b16(svptrue_b16(), equal_mask);
1365
- }
1366
-
1367
- // Advance
1368
- a_idx += a_step;
1369
- b_idx += b_step;
1370
- }
1371
- results[0] = (simsimd_distance_t)intersection_size;
1372
- results[1] = svaddv_f32(svptrue_b32(), product_vec);
1373
- }
1374
-
1375
- #pragma clang attribute pop
1376
- #pragma GCC pop_options
1377
- #endif // SIMSIMD_TARGET_SVE2 && SIMSIMD_TARGET_SVE_BF16
1378
- #endif // _SIMSIMD_TARGET_ARM
1379
-
1380
- #ifdef __cplusplus
1381
- }
1382
- #endif
1383
-
1384
- #endif