usearch 2.21.3 → 2.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (602) hide show
  1. package/README.md +24 -24
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +7 -7
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +83 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +235 -52
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/dist/cjs/usearch.d.ts +8 -3
  16. package/javascript/dist/cjs/usearch.js +12 -4
  17. package/javascript/dist/esm/usearch.d.ts +8 -3
  18. package/javascript/dist/esm/usearch.js +12 -4
  19. package/javascript/lib.cpp +17 -0
  20. package/javascript/usearch.ts +14 -6
  21. package/numkong/.clang-tidy +30 -0
  22. package/{simsimd → numkong}/.cmake-format.py +18 -0
  23. package/numkong/.editorconfig +12 -0
  24. package/numkong/.git-blame-ignore-revs +123 -0
  25. package/numkong/.mailmap +15 -0
  26. package/numkong/.spi.yml +11 -0
  27. package/numkong/.swift-format +14 -0
  28. package/numkong/CITATION.cff +81 -0
  29. package/numkong/CONTRIBUTING.md +500 -0
  30. package/numkong/Cargo.toml +54 -0
  31. package/numkong/MANIFEST.in +8 -0
  32. package/numkong/Package.swift +84 -0
  33. package/numkong/README.md +569 -0
  34. package/numkong/bench/README.md +217 -0
  35. package/numkong/bench/bench-browser.html +323 -0
  36. package/numkong/bench/bench.cpp +344 -0
  37. package/numkong/bench/bench.hpp +816 -0
  38. package/numkong/bench/bench.js +148 -0
  39. package/numkong/bench/bench.mjs +616 -0
  40. package/numkong/bench/bench_attention.cpp +113 -0
  41. package/numkong/bench/bench_cast.cpp +264 -0
  42. package/numkong/bench/bench_cross_amx.cpp +62 -0
  43. package/numkong/bench/bench_cross_arm.cpp +245 -0
  44. package/numkong/bench/bench_cross_blas.cpp +309 -0
  45. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  46. package/numkong/bench/bench_cross_power.cpp +84 -0
  47. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  48. package/numkong/bench/bench_cross_serial.cpp +151 -0
  49. package/numkong/bench/bench_cross_sme.cpp +163 -0
  50. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  51. package/numkong/bench/bench_cross_x86.cpp +455 -0
  52. package/numkong/bench/bench_curved.cpp +210 -0
  53. package/numkong/bench/bench_dot.cpp +315 -0
  54. package/numkong/bench/bench_each.cpp +503 -0
  55. package/numkong/bench/bench_geospatial.cpp +112 -0
  56. package/numkong/bench/bench_maxsim.cpp +154 -0
  57. package/numkong/bench/bench_mesh.cpp +148 -0
  58. package/numkong/bench/bench_probability.cpp +60 -0
  59. package/numkong/bench/bench_reduce.cpp +371 -0
  60. package/numkong/bench/bench_set.cpp +73 -0
  61. package/numkong/bench/bench_sparse.cpp +198 -0
  62. package/numkong/bench/bench_spatial.cpp +347 -0
  63. package/numkong/bench/bench_trigonometry.cpp +137 -0
  64. package/numkong/bench/swift/Bench.swift +506 -0
  65. package/numkong/binding.gyp +169 -0
  66. package/numkong/build.rs +484 -0
  67. package/numkong/c/dispatch.h +512 -0
  68. package/numkong/c/dispatch_bf16.c +447 -0
  69. package/numkong/c/dispatch_bf16c.c +52 -0
  70. package/numkong/c/dispatch_e2m3.c +296 -0
  71. package/numkong/c/dispatch_e3m2.c +284 -0
  72. package/numkong/c/dispatch_e4m3.c +318 -0
  73. package/numkong/c/dispatch_e5m2.c +316 -0
  74. package/numkong/c/dispatch_f16.c +452 -0
  75. package/numkong/c/dispatch_f16c.c +58 -0
  76. package/numkong/c/dispatch_f32.c +434 -0
  77. package/numkong/c/dispatch_f32c.c +99 -0
  78. package/numkong/c/dispatch_f64.c +348 -0
  79. package/numkong/c/dispatch_f64c.c +98 -0
  80. package/numkong/c/dispatch_i16.c +96 -0
  81. package/numkong/c/dispatch_i32.c +89 -0
  82. package/numkong/c/dispatch_i4.c +153 -0
  83. package/numkong/c/dispatch_i64.c +86 -0
  84. package/numkong/c/dispatch_i8.c +348 -0
  85. package/numkong/c/dispatch_other.c +348 -0
  86. package/numkong/c/dispatch_u1.c +193 -0
  87. package/numkong/c/dispatch_u16.c +124 -0
  88. package/numkong/c/dispatch_u32.c +118 -0
  89. package/numkong/c/dispatch_u4.c +153 -0
  90. package/numkong/c/dispatch_u64.c +102 -0
  91. package/numkong/c/dispatch_u8.c +364 -0
  92. package/numkong/c/numkong.c +940 -0
  93. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  94. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  95. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  96. package/numkong/cmake/nk_math_lib.cmake +46 -0
  97. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  99. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  100. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  101. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  102. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  103. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  104. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  105. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  106. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  107. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  108. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  109. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  110. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  111. package/numkong/include/README.md +643 -0
  112. package/numkong/include/module.modulemap +129 -0
  113. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  114. package/numkong/include/numkong/attention/sme.h +2075 -0
  115. package/numkong/include/numkong/attention.h +49 -0
  116. package/numkong/include/numkong/capabilities.h +930 -0
  117. package/numkong/include/numkong/cast/README.md +262 -0
  118. package/numkong/include/numkong/cast/diamond.h +64 -0
  119. package/numkong/include/numkong/cast/haswell.h +823 -0
  120. package/numkong/include/numkong/cast/icelake.h +475 -0
  121. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  122. package/numkong/include/numkong/cast/neon.h +1174 -0
  123. package/numkong/include/numkong/cast/powervsx.h +449 -0
  124. package/numkong/include/numkong/cast/rvv.h +970 -0
  125. package/numkong/include/numkong/cast/sapphire.h +262 -0
  126. package/numkong/include/numkong/cast/serial.h +2336 -0
  127. package/numkong/include/numkong/cast/skylake.h +915 -0
  128. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  129. package/numkong/include/numkong/cast.h +254 -0
  130. package/numkong/include/numkong/cast.hpp +44 -0
  131. package/numkong/include/numkong/curved/README.md +223 -0
  132. package/numkong/include/numkong/curved/genoa.h +182 -0
  133. package/numkong/include/numkong/curved/haswell.h +276 -0
  134. package/numkong/include/numkong/curved/neon.h +329 -0
  135. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  136. package/numkong/include/numkong/curved/rvv.h +305 -0
  137. package/numkong/include/numkong/curved/serial.h +207 -0
  138. package/numkong/include/numkong/curved/skylake.h +457 -0
  139. package/numkong/include/numkong/curved/smef64.h +530 -0
  140. package/numkong/include/numkong/curved.h +513 -0
  141. package/numkong/include/numkong/curved.hpp +144 -0
  142. package/numkong/include/numkong/dot/README.md +442 -0
  143. package/numkong/include/numkong/dot/alder.h +563 -0
  144. package/numkong/include/numkong/dot/diamond.h +158 -0
  145. package/numkong/include/numkong/dot/genoa.h +289 -0
  146. package/numkong/include/numkong/dot/haswell.h +1723 -0
  147. package/numkong/include/numkong/dot/icelake.h +997 -0
  148. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  149. package/numkong/include/numkong/dot/neon.h +869 -0
  150. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  151. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  152. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  153. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  154. package/numkong/include/numkong/dot/powervsx.h +752 -0
  155. package/numkong/include/numkong/dot/rvv.h +722 -0
  156. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  157. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  158. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  159. package/numkong/include/numkong/dot/sapphire.h +141 -0
  160. package/numkong/include/numkong/dot/serial.h +874 -0
  161. package/numkong/include/numkong/dot/sierra.h +405 -0
  162. package/numkong/include/numkong/dot/skylake.h +1116 -0
  163. package/numkong/include/numkong/dot/sve.h +420 -0
  164. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  165. package/numkong/include/numkong/dot/svehalf.h +168 -0
  166. package/numkong/include/numkong/dot/svesdot.h +90 -0
  167. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  168. package/numkong/include/numkong/dot.h +1136 -0
  169. package/numkong/include/numkong/dot.hpp +94 -0
  170. package/numkong/include/numkong/dots/README.md +496 -0
  171. package/numkong/include/numkong/dots/alder.h +117 -0
  172. package/numkong/include/numkong/dots/diamond.h +86 -0
  173. package/numkong/include/numkong/dots/genoa.h +100 -0
  174. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  175. package/numkong/include/numkong/dots/haswell.h +310 -0
  176. package/numkong/include/numkong/dots/icelake.h +180 -0
  177. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  178. package/numkong/include/numkong/dots/neon.h +123 -0
  179. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  180. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  181. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  182. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  183. package/numkong/include/numkong/dots/powervsx.h +194 -0
  184. package/numkong/include/numkong/dots/rvv.h +2593 -0
  185. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  186. package/numkong/include/numkong/dots/serial.h +2867 -0
  187. package/numkong/include/numkong/dots/sierra.h +100 -0
  188. package/numkong/include/numkong/dots/skylake.h +207 -0
  189. package/numkong/include/numkong/dots/sme.h +5041 -0
  190. package/numkong/include/numkong/dots/smebi32.h +477 -0
  191. package/numkong/include/numkong/dots/smef64.h +1328 -0
  192. package/numkong/include/numkong/dots/smehalf.h +47 -0
  193. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  194. package/numkong/include/numkong/dots.h +3110 -0
  195. package/numkong/include/numkong/dots.hpp +641 -0
  196. package/numkong/include/numkong/each/README.md +465 -0
  197. package/numkong/include/numkong/each/haswell.h +1658 -0
  198. package/numkong/include/numkong/each/icelake.h +272 -0
  199. package/numkong/include/numkong/each/neon.h +1138 -0
  200. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  201. package/numkong/include/numkong/each/neonhalf.h +318 -0
  202. package/numkong/include/numkong/each/rvv.h +1121 -0
  203. package/numkong/include/numkong/each/sapphire.h +343 -0
  204. package/numkong/include/numkong/each/serial.h +282 -0
  205. package/numkong/include/numkong/each/skylake.h +1562 -0
  206. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  207. package/numkong/include/numkong/each.h +2232 -0
  208. package/numkong/include/numkong/each.hpp +434 -0
  209. package/numkong/include/numkong/geospatial/README.md +147 -0
  210. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  211. package/numkong/include/numkong/geospatial/neon.h +615 -0
  212. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  213. package/numkong/include/numkong/geospatial/serial.h +309 -0
  214. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  215. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  216. package/numkong/include/numkong/geospatial.h +453 -0
  217. package/numkong/include/numkong/geospatial.hpp +235 -0
  218. package/numkong/include/numkong/matrix.hpp +336 -0
  219. package/numkong/include/numkong/maxsim/README.md +187 -0
  220. package/numkong/include/numkong/maxsim/alder.h +511 -0
  221. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  222. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  223. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  224. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  225. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  226. package/numkong/include/numkong/maxsim/serial.h +490 -0
  227. package/numkong/include/numkong/maxsim/sme.h +993 -0
  228. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  229. package/numkong/include/numkong/maxsim.h +571 -0
  230. package/numkong/include/numkong/maxsim.hpp +133 -0
  231. package/numkong/include/numkong/mesh/README.md +219 -0
  232. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  233. package/numkong/include/numkong/mesh/neon.h +1821 -0
  234. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  235. package/numkong/include/numkong/mesh/rvv.h +936 -0
  236. package/numkong/include/numkong/mesh/serial.h +721 -0
  237. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  238. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  239. package/numkong/include/numkong/mesh.h +666 -0
  240. package/numkong/include/numkong/mesh.hpp +701 -0
  241. package/numkong/include/numkong/numkong.h +78 -0
  242. package/numkong/include/numkong/numkong.hpp +58 -0
  243. package/numkong/include/numkong/probability/README.md +177 -0
  244. package/numkong/include/numkong/probability/haswell.h +267 -0
  245. package/numkong/include/numkong/probability/neon.h +255 -0
  246. package/numkong/include/numkong/probability/rvv.h +414 -0
  247. package/numkong/include/numkong/probability/serial.h +181 -0
  248. package/numkong/include/numkong/probability/skylake.h +324 -0
  249. package/numkong/include/numkong/probability.h +380 -0
  250. package/numkong/include/numkong/probability.hpp +120 -0
  251. package/numkong/include/numkong/random.h +50 -0
  252. package/numkong/include/numkong/random.hpp +285 -0
  253. package/numkong/include/numkong/reduce/README.md +552 -0
  254. package/numkong/include/numkong/reduce/alder.h +636 -0
  255. package/numkong/include/numkong/reduce/genoa.h +201 -0
  256. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  257. package/numkong/include/numkong/reduce/icelake.h +549 -0
  258. package/numkong/include/numkong/reduce/neon.h +3980 -0
  259. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  260. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  261. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  262. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  263. package/numkong/include/numkong/reduce/serial.h +753 -0
  264. package/numkong/include/numkong/reduce/sierra.h +338 -0
  265. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  266. package/numkong/include/numkong/reduce/sve.h +52 -0
  267. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  268. package/numkong/include/numkong/reduce.h +1582 -0
  269. package/numkong/include/numkong/reduce.hpp +778 -0
  270. package/numkong/include/numkong/scalar/README.md +89 -0
  271. package/numkong/include/numkong/scalar/haswell.h +122 -0
  272. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  273. package/numkong/include/numkong/scalar/neon.h +122 -0
  274. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  275. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  276. package/numkong/include/numkong/scalar/rvv.h +211 -0
  277. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  278. package/numkong/include/numkong/scalar/serial.h +332 -0
  279. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  280. package/numkong/include/numkong/scalar.h +696 -0
  281. package/numkong/include/numkong/set/README.md +179 -0
  282. package/numkong/include/numkong/set/haswell.h +334 -0
  283. package/numkong/include/numkong/set/icelake.h +485 -0
  284. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  285. package/numkong/include/numkong/set/neon.h +363 -0
  286. package/numkong/include/numkong/set/powervsx.h +326 -0
  287. package/numkong/include/numkong/set/rvv.h +226 -0
  288. package/numkong/include/numkong/set/rvvbb.h +117 -0
  289. package/numkong/include/numkong/set/serial.h +174 -0
  290. package/numkong/include/numkong/set/sve.h +187 -0
  291. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  292. package/numkong/include/numkong/set.h +471 -0
  293. package/numkong/include/numkong/set.hpp +114 -0
  294. package/numkong/include/numkong/sets/README.md +149 -0
  295. package/numkong/include/numkong/sets/haswell.h +63 -0
  296. package/numkong/include/numkong/sets/icelake.h +66 -0
  297. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  298. package/numkong/include/numkong/sets/neon.h +61 -0
  299. package/numkong/include/numkong/sets/powervsx.h +65 -0
  300. package/numkong/include/numkong/sets/serial.h +43 -0
  301. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  302. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  303. package/numkong/include/numkong/sets.h +382 -0
  304. package/numkong/include/numkong/sparse/README.md +156 -0
  305. package/numkong/include/numkong/sparse/icelake.h +463 -0
  306. package/numkong/include/numkong/sparse/neon.h +288 -0
  307. package/numkong/include/numkong/sparse/serial.h +117 -0
  308. package/numkong/include/numkong/sparse/sve2.h +504 -0
  309. package/numkong/include/numkong/sparse/turin.h +322 -0
  310. package/numkong/include/numkong/sparse.h +363 -0
  311. package/numkong/include/numkong/sparse.hpp +113 -0
  312. package/numkong/include/numkong/spatial/README.md +439 -0
  313. package/numkong/include/numkong/spatial/alder.h +604 -0
  314. package/numkong/include/numkong/spatial/diamond.h +240 -0
  315. package/numkong/include/numkong/spatial/genoa.h +222 -0
  316. package/numkong/include/numkong/spatial/haswell.h +979 -0
  317. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  318. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  319. package/numkong/include/numkong/spatial/neon.h +846 -0
  320. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  321. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  322. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  323. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  324. package/numkong/include/numkong/spatial/rvv.h +991 -0
  325. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  326. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  327. package/numkong/include/numkong/spatial/serial.h +347 -0
  328. package/numkong/include/numkong/spatial/sierra.h +516 -0
  329. package/numkong/include/numkong/spatial/skylake.h +605 -0
  330. package/numkong/include/numkong/spatial/sve.h +246 -0
  331. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  332. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  333. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  334. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  335. package/numkong/include/numkong/spatial.h +1681 -0
  336. package/numkong/include/numkong/spatial.hpp +183 -0
  337. package/numkong/include/numkong/spatials/README.md +580 -0
  338. package/numkong/include/numkong/spatials/alder.h +94 -0
  339. package/numkong/include/numkong/spatials/diamond.h +82 -0
  340. package/numkong/include/numkong/spatials/genoa.h +94 -0
  341. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  342. package/numkong/include/numkong/spatials/haswell.h +219 -0
  343. package/numkong/include/numkong/spatials/icelake.h +113 -0
  344. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  345. package/numkong/include/numkong/spatials/neon.h +109 -0
  346. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  347. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  348. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  349. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  350. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  351. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  352. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  353. package/numkong/include/numkong/spatials/serial.h +248 -0
  354. package/numkong/include/numkong/spatials/sierra.h +96 -0
  355. package/numkong/include/numkong/spatials/skylake.h +184 -0
  356. package/numkong/include/numkong/spatials/sme.h +1927 -0
  357. package/numkong/include/numkong/spatials/smef64.h +483 -0
  358. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  359. package/numkong/include/numkong/spatials.h +3374 -0
  360. package/numkong/include/numkong/spatials.hpp +509 -0
  361. package/numkong/include/numkong/tensor.hpp +1981 -0
  362. package/numkong/include/numkong/trigonometry/README.md +184 -0
  363. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  364. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  365. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  366. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  367. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  368. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  369. package/numkong/include/numkong/trigonometry.h +467 -0
  370. package/numkong/include/numkong/trigonometry.hpp +166 -0
  371. package/numkong/include/numkong/types.h +1695 -0
  372. package/numkong/include/numkong/types.hpp +6027 -0
  373. package/numkong/include/numkong/vector.hpp +766 -0
  374. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  375. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  376. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  377. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  378. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  380. package/numkong/javascript/README.md +254 -0
  381. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  382. package/numkong/javascript/numkong-browser.ts +40 -0
  383. package/numkong/javascript/numkong-wasm.ts +750 -0
  384. package/numkong/javascript/numkong.c +712 -0
  385. package/numkong/javascript/numkong.ts +613 -0
  386. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  387. package/numkong/javascript/tsconfig-cjs.json +13 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  389. package/numkong/javascript/types.ts +784 -0
  390. package/numkong/numkong.gypi +104 -0
  391. package/numkong/package.json +108 -0
  392. package/numkong/probes/arm_neon.c +8 -0
  393. package/numkong/probes/arm_neon_bfdot.c +9 -0
  394. package/numkong/probes/arm_neon_fhm.c +9 -0
  395. package/numkong/probes/arm_neon_half.c +8 -0
  396. package/numkong/probes/arm_neon_sdot.c +9 -0
  397. package/numkong/probes/arm_neonfp8.c +9 -0
  398. package/numkong/probes/arm_sme.c +16 -0
  399. package/numkong/probes/arm_sme2.c +16 -0
  400. package/numkong/probes/arm_sme2p1.c +16 -0
  401. package/numkong/probes/arm_sme_bf16.c +16 -0
  402. package/numkong/probes/arm_sme_bi32.c +16 -0
  403. package/numkong/probes/arm_sme_f64.c +16 -0
  404. package/numkong/probes/arm_sme_fa64.c +14 -0
  405. package/numkong/probes/arm_sme_half.c +16 -0
  406. package/numkong/probes/arm_sme_lut2.c +15 -0
  407. package/numkong/probes/arm_sve.c +18 -0
  408. package/numkong/probes/arm_sve2.c +20 -0
  409. package/numkong/probes/arm_sve2p1.c +18 -0
  410. package/numkong/probes/arm_sve_bfdot.c +20 -0
  411. package/numkong/probes/arm_sve_half.c +18 -0
  412. package/numkong/probes/arm_sve_sdot.c +21 -0
  413. package/numkong/probes/loongarch_lasx.c +12 -0
  414. package/numkong/probes/power_vsx.c +12 -0
  415. package/numkong/probes/probe.js +127 -0
  416. package/numkong/probes/riscv_rvv.c +14 -0
  417. package/numkong/probes/riscv_rvv_bb.c +15 -0
  418. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  419. package/numkong/probes/riscv_rvv_half.c +14 -0
  420. package/numkong/probes/wasm_v128relaxed.c +11 -0
  421. package/numkong/probes/x86_alder.c +17 -0
  422. package/numkong/probes/x86_diamond.c +17 -0
  423. package/numkong/probes/x86_genoa.c +17 -0
  424. package/numkong/probes/x86_graniteamx.c +19 -0
  425. package/numkong/probes/x86_haswell.c +11 -0
  426. package/numkong/probes/x86_icelake.c +17 -0
  427. package/numkong/probes/x86_sapphire.c +16 -0
  428. package/numkong/probes/x86_sapphireamx.c +18 -0
  429. package/numkong/probes/x86_sierra.c +17 -0
  430. package/numkong/probes/x86_skylake.c +15 -0
  431. package/numkong/probes/x86_turin.c +17 -0
  432. package/numkong/rust/README.md +698 -0
  433. package/numkong/rust/capabilities.rs +96 -0
  434. package/numkong/rust/cast.rs +314 -0
  435. package/numkong/rust/curved.rs +391 -0
  436. package/numkong/rust/each.rs +2721 -0
  437. package/numkong/rust/geospatial.rs +343 -0
  438. package/numkong/rust/matrix.rs +5424 -0
  439. package/numkong/rust/maxsim.rs +540 -0
  440. package/numkong/rust/mesh.rs +837 -0
  441. package/numkong/rust/numkong.rs +409 -0
  442. package/numkong/rust/probability.rs +272 -0
  443. package/numkong/rust/reduce.rs +1249 -0
  444. package/numkong/rust/set.rs +194 -0
  445. package/numkong/rust/sparse.rs +513 -0
  446. package/numkong/rust/spatial.rs +1815 -0
  447. package/numkong/rust/tensor.rs +7565 -0
  448. package/numkong/rust/types.rs +3666 -0
  449. package/numkong/rust/vector.rs +1899 -0
  450. package/numkong/rustfmt.toml +2 -0
  451. package/numkong/swift/Matrix.swift +1388 -0
  452. package/numkong/swift/MaxSim.swift +151 -0
  453. package/numkong/swift/NumKong.swift +283 -0
  454. package/numkong/swift/Numerics.swift +329 -0
  455. package/numkong/swift/README.md +433 -0
  456. package/numkong/swift/Scalar.swift +741 -0
  457. package/numkong/swift/Tensor.swift +209 -0
  458. package/numkong/test/README.md +315 -0
  459. package/numkong/test/playwright.config.ts +16 -0
  460. package/numkong/test/swift/Test.swift +534 -0
  461. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  462. package/numkong/test/test-browser-bundle.html +188 -0
  463. package/numkong/test/test-browser-runner.mjs +43 -0
  464. package/numkong/test/test-browser.html +305 -0
  465. package/numkong/test/test-dtypes.mjs +453 -0
  466. package/numkong/test/test-wasm.mjs +267 -0
  467. package/numkong/test/test.cpp +442 -0
  468. package/numkong/test/test.hpp +580 -0
  469. package/numkong/test/test.mjs +363 -0
  470. package/numkong/test/test_base.py +1041 -0
  471. package/numkong/test/test_cast.cpp +242 -0
  472. package/numkong/test/test_cross.hpp +551 -0
  473. package/numkong/test/test_cross.py +467 -0
  474. package/numkong/test/test_cross_amx.cpp +50 -0
  475. package/numkong/test/test_cross_arm.cpp +225 -0
  476. package/numkong/test/test_cross_blas.cpp +354 -0
  477. package/numkong/test/test_cross_loongarch.cpp +90 -0
  478. package/numkong/test/test_cross_power.cpp +73 -0
  479. package/numkong/test/test_cross_rvv.cpp +109 -0
  480. package/numkong/test/test_cross_serial.cpp +181 -0
  481. package/numkong/test/test_cross_sme.cpp +148 -0
  482. package/numkong/test/test_cross_wasm.cpp +141 -0
  483. package/numkong/test/test_cross_x86.cpp +442 -0
  484. package/numkong/test/test_curved.cpp +182 -0
  485. package/numkong/test/test_curved.py +207 -0
  486. package/numkong/test/test_dot.cpp +353 -0
  487. package/numkong/test/test_dot.py +296 -0
  488. package/numkong/test/test_each.cpp +437 -0
  489. package/numkong/test/test_each.py +756 -0
  490. package/numkong/test/test_geospatial.cpp +155 -0
  491. package/numkong/test/test_geospatial.py +241 -0
  492. package/numkong/test/test_maxsim.cpp +138 -0
  493. package/numkong/test/test_maxsim.py +209 -0
  494. package/numkong/test/test_mesh.cpp +203 -0
  495. package/numkong/test/test_mesh.py +394 -0
  496. package/numkong/test/test_probability.cpp +136 -0
  497. package/numkong/test/test_probability.py +166 -0
  498. package/numkong/test/test_reduce.cpp +563 -0
  499. package/numkong/test/test_reduce.py +781 -0
  500. package/numkong/test/test_set.cpp +151 -0
  501. package/numkong/test/test_set.py +120 -0
  502. package/numkong/test/test_similarities.py +579 -0
  503. package/numkong/test/test_sparse.cpp +121 -0
  504. package/numkong/test/test_sparse.py +118 -0
  505. package/numkong/test/test_spatial.cpp +518 -0
  506. package/numkong/test/test_spatial.py +269 -0
  507. package/numkong/test/test_tensor.cpp +884 -0
  508. package/numkong/test/test_tensor.py +1417 -0
  509. package/numkong/test/test_trigonometry.cpp +156 -0
  510. package/numkong/test/test_trigonometry.py +180 -0
  511. package/package.json +7 -8
  512. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  513. package/prebuilds/linux-arm64/usearch.node +0 -0
  514. package/prebuilds/linux-x64/usearch.node +0 -0
  515. package/fp16/CMakeLists.txt +0 -176
  516. package/fp16/LICENSE +0 -11
  517. package/fp16/README.md +0 -20
  518. package/fp16/bench/alt-element.cc +0 -89
  519. package/fp16/bench/from-alt-array.cc +0 -175
  520. package/fp16/bench/from-ieee-array.cc +0 -399
  521. package/fp16/bench/ieee-element.cc +0 -272
  522. package/fp16/bench/to-alt-array.cc +0 -91
  523. package/fp16/bench/to-ieee-array.cc +0 -303
  524. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  525. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  526. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  527. package/fp16/configure.py +0 -65
  528. package/fp16/confu.yaml +0 -8
  529. package/fp16/include/fp16/__init__.py +0 -0
  530. package/fp16/include/fp16/avx.py +0 -50
  531. package/fp16/include/fp16/avx2.py +0 -53
  532. package/fp16/include/fp16/bitcasts.h +0 -92
  533. package/fp16/include/fp16/fp16.h +0 -451
  534. package/fp16/include/fp16/psimd.h +0 -131
  535. package/fp16/include/fp16.h +0 -11
  536. package/fp16/jni/Android.mk +0 -6
  537. package/fp16/jni/Application.mk +0 -4
  538. package/fp16/test/alt-from-fp32-value.cc +0 -498
  539. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  540. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  541. package/fp16/test/alt-to-fp32-value.cc +0 -348
  542. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  543. package/fp16/test/bitcasts.cc +0 -56
  544. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  545. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  546. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  547. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  548. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  549. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  550. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  551. package/fp16/test/peachpy/stubs.py +0 -38
  552. package/fp16/test/tables.cc +0 -530
  553. package/fp16/test/tables.h +0 -13
  554. package/fp16/third-party/THHalf.h +0 -92
  555. package/fp16/third-party/eigen-half.h +0 -170
  556. package/fp16/third-party/float16-compressor.h +0 -80
  557. package/fp16/third-party/half.hpp +0 -2916
  558. package/fp16/third-party/npy-halffloat.h +0 -328
  559. package/simsimd/.git-blame-ignore-revs +0 -2
  560. package/simsimd/CONTRIBUTING.md +0 -321
  561. package/simsimd/Cargo.toml +0 -49
  562. package/simsimd/MANIFEST.in +0 -4
  563. package/simsimd/Package.swift +0 -28
  564. package/simsimd/README.md +0 -1493
  565. package/simsimd/binding.gyp +0 -35
  566. package/simsimd/build.rs +0 -70
  567. package/simsimd/c/lib.c +0 -374
  568. package/simsimd/include/module.modulemap +0 -4
  569. package/simsimd/include/simsimd/binary.h +0 -486
  570. package/simsimd/include/simsimd/curved.h +0 -1541
  571. package/simsimd/include/simsimd/dot.h +0 -1853
  572. package/simsimd/include/simsimd/elementwise.h +0 -2465
  573. package/simsimd/include/simsimd/geospatial.h +0 -43
  574. package/simsimd/include/simsimd/mesh.h +0 -69
  575. package/simsimd/include/simsimd/probability.h +0 -606
  576. package/simsimd/include/simsimd/simsimd.h +0 -2567
  577. package/simsimd/include/simsimd/sparse.h +0 -1384
  578. package/simsimd/include/simsimd/spatial.h +0 -2335
  579. package/simsimd/include/simsimd/types.h +0 -666
  580. package/simsimd/javascript/fallback.ts +0 -233
  581. package/simsimd/javascript/lib.c +0 -125
  582. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  583. package/simsimd/javascript/simsimd.ts +0 -179
  584. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  585. package/simsimd/package-lock.json +0 -1761
  586. package/simsimd/package.json +0 -61
  587. package/simsimd/rust/lib.rs +0 -1821
  588. package/simsimd/scripts/bench.cxx +0 -1170
  589. package/simsimd/scripts/bench.ipynb +0 -187
  590. package/simsimd/scripts/bench.js +0 -127
  591. package/simsimd/scripts/bench.rs +0 -876
  592. package/simsimd/scripts/bench_vectors.py +0 -734
  593. package/simsimd/scripts/bench_vectors_live.py +0 -213
  594. package/simsimd/scripts/test.c +0 -187
  595. package/simsimd/scripts/test.mjs +0 -220
  596. package/simsimd/scripts/test.py +0 -1505
  597. package/simsimd/swift/SimSIMD.swift +0 -149
  598. package/simsimd/swift/Test.swift +0 -128
  599. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  600. /package/{simsimd → numkong}/LICENSE +0 -0
  601. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  602. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -1,1853 +0,0 @@
1
- /**
2
- * @file dot.h
3
- * @brief SIMD-accelerated Dot Products for Real and Complex numbers.
4
- * @author Ash Vardanian
5
- * @date February 24, 2024
6
- *
7
- * Contains:
8
- * - Dot Product for Real and Complex vectors
9
- * - Conjugate Dot Product for Complex vectors
10
- *
11
- * For datatypes:
12
- * - 64-bit IEEE floating point numbers
13
- * - 32-bit IEEE floating point numbers
14
- * - 16-bit IEEE floating point numbers
15
- * - 16-bit brain floating point numbers
16
- * - 8-bit unsigned integers
17
- * - 8-bit signed integers
18
- *
19
- * For hardware architectures:
20
- * - Arm: NEON, SVE
21
- * - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
22
- *
23
- * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
24
- * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
25
- */
26
- #ifndef SIMSIMD_DOT_H
27
- #define SIMSIMD_DOT_H
28
-
29
- #include "types.h"
30
-
31
- #ifdef __cplusplus
32
- extern "C" {
33
- #endif
34
-
35
- // clang-format off
36
-
37
- /* Serial backends for all numeric types.
38
- * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
39
- * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
40
- */
41
- SIMSIMD_PUBLIC void simsimd_dot_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
42
- SIMSIMD_PUBLIC void simsimd_dot_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
43
- SIMSIMD_PUBLIC void simsimd_vdot_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
44
-
45
- SIMSIMD_PUBLIC void simsimd_dot_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
46
- SIMSIMD_PUBLIC void simsimd_dot_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
47
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
48
-
49
- SIMSIMD_PUBLIC void simsimd_dot_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
50
- SIMSIMD_PUBLIC void simsimd_dot_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
51
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
52
-
53
- SIMSIMD_PUBLIC void simsimd_dot_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
54
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
55
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
56
-
57
- SIMSIMD_PUBLIC void simsimd_dot_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
58
- SIMSIMD_PUBLIC void simsimd_dot_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
59
-
60
- /* Double-precision serial backends for all numeric types.
61
- * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
62
- */
63
- SIMSIMD_PUBLIC void simsimd_dot_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
64
- SIMSIMD_PUBLIC void simsimd_dot_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
65
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
66
-
67
- SIMSIMD_PUBLIC void simsimd_dot_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
68
- SIMSIMD_PUBLIC void simsimd_dot_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
69
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
70
-
71
- SIMSIMD_PUBLIC void simsimd_dot_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
72
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
73
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
74
-
75
- /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
76
- * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
77
- * server CPUs produced before 2023.
78
- */
79
- SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
80
- SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
81
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
82
-
83
- SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
84
- SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
85
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
86
-
87
- SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
88
- SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
89
-
90
- SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
91
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
92
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
93
-
94
- /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
95
- * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
96
- */
97
- SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
98
- SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
99
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
100
-
101
- SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
102
- SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
103
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
104
-
105
- SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
106
- SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
107
- SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
108
-
109
- /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
110
- * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
111
- * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
112
- * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
113
- * properly vectorized by recent compilers.
114
- */
115
- SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
116
- SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
117
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
118
-
119
- SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
120
- SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
121
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
122
-
123
- SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
124
-
125
- SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
126
- SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
127
-
128
- /* SIMD-powered backends for various generations of AVX512 CPUs.
129
- * Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
130
- * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
131
- * Genoa added only BF16.
132
- * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
133
- *
134
- * Sadly, we can't effectively interleave different kinds of arithmetic instructions to utilize more ports:
135
- *
136
- * > Like Intel server architectures since Skylake-X, SPR cores feature two 512-bit FMA units, and organize them in a similar fashion.
137
- * > One 512-bit FMA unit is created by fusing two 256-bit ones on port 0 and port 1. The other is added to port 5, as a server-specific
138
- * > core extension. The FMA units on port 0 and 1 are configured into 2×256-bit or 1×512-bit mode depending on whether 512-bit FMA
139
- * > instructions are present in the scheduler. That means a mix of 256-bit and 512-bit FMA instructions will not achieve higher IPC
140
- * > than executing 512-bit instructions alone.
141
- *
142
- * Source: https://chipsandcheese.com/p/a-peek-at-sapphire-rapids
143
- */
144
- SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
145
- SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
146
- SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
147
-
148
- SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
149
- SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
150
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
151
-
152
- SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
153
- SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
154
-
155
- SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
156
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
157
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
158
-
159
- SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
160
- SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
161
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_size_t n, simsimd_distance_t* results);
162
-
163
- SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
164
- // clang-format on
165
-
166
- #define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert) \
167
- SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const *a, \
168
- simsimd_##input_type##_t const *b, simsimd_size_t n, \
169
- simsimd_distance_t *result) { \
170
- simsimd_##accumulator_type##_t ab = 0; \
171
- for (simsimd_size_t i = 0; i != n; ++i) { \
172
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
173
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
174
- ab += ai * bi; \
175
- } \
176
- *result = ab; \
177
- }
178
-
179
- #define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert) \
180
- SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const *a_pairs, \
181
- simsimd_##input_type##_t const *b_pairs, \
182
- simsimd_size_t count_pairs, simsimd_distance_t *results) { \
183
- simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
184
- for (simsimd_size_t i = 0; i != count_pairs; ++i) { \
185
- simsimd_##accumulator_type##_t ar = load_and_convert(&(a_pairs + i)->real); \
186
- simsimd_##accumulator_type##_t br = load_and_convert(&(b_pairs + i)->real); \
187
- simsimd_##accumulator_type##_t ai = load_and_convert(&(a_pairs + i)->imag); \
188
- simsimd_##accumulator_type##_t bi = load_and_convert(&(b_pairs + i)->imag); \
189
- ab_real += ar * br - ai * bi; \
190
- ab_imag += ar * bi + ai * br; \
191
- } \
192
- results[0] = ab_real; \
193
- results[1] = ab_imag; \
194
- }
195
-
196
- #define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert) \
197
- SIMSIMD_PUBLIC void simsimd_vdot_##input_type##_##name(simsimd_##input_type##_t const *a_pairs, \
198
- simsimd_##input_type##_t const *b_pairs, \
199
- simsimd_size_t count_pairs, simsimd_distance_t *results) { \
200
- simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
201
- for (simsimd_size_t i = 0; i != count_pairs; ++i) { \
202
- simsimd_##accumulator_type##_t ar = load_and_convert(&(a_pairs + i)->real); \
203
- simsimd_##accumulator_type##_t br = load_and_convert(&(b_pairs + i)->real); \
204
- simsimd_##accumulator_type##_t ai = load_and_convert(&(a_pairs + i)->imag); \
205
- simsimd_##accumulator_type##_t bi = load_and_convert(&(b_pairs + i)->imag); \
206
- ab_real += ar * br + ai * bi; \
207
- ab_imag += ar * bi - ai * br; \
208
- } \
209
- results[0] = ab_real; \
210
- results[1] = ab_imag; \
211
- }
212
-
213
- SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64_serial
214
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64c_serial
215
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f64c_serial
216
-
217
- SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_serial
218
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_serial
219
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_serial
220
-
221
- SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_serial
222
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_serial
223
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_serial
224
-
225
- SIMSIMD_MAKE_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_serial
226
- SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_serial
227
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_serial
228
-
229
- SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_i8_serial
230
- SIMSIMD_MAKE_DOT(serial, u8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_u8_serial
231
-
232
- SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_accurate
233
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_accurate
234
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_accurate
235
-
236
- SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_accurate
237
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_accurate
238
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_accurate
239
-
240
- SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_accurate
241
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_accurate
242
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_accurate
243
-
244
- #if _SIMSIMD_TARGET_ARM
245
- #if SIMSIMD_TARGET_NEON
246
- #pragma GCC push_options
247
- #pragma GCC target("arch=armv8-a+simd")
248
- #pragma clang attribute push(__attribute__((target("arch=armv8-a+simd"))), apply_to = function)
249
-
250
- SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const *x, simsimd_size_t n) {
251
- union {
252
- float32x4_t vec;
253
- simsimd_f32_t scalars[4];
254
- } result;
255
- simsimd_size_t i = 0;
256
- for (; i < n; ++i) result.scalars[i] = x[i];
257
- for (; i < 4; ++i) result.scalars[i] = 0;
258
- return result.vec;
259
- }
260
-
261
- SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const *a_scalars, simsimd_f32_t const *b_scalars,
262
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
263
- float32x4_t ab_vec = vdupq_n_f32(0);
264
- simsimd_size_t idx_scalars = 0;
265
- for (; idx_scalars + 4 <= count_scalars; idx_scalars += 4) {
266
- float32x4_t a_vec = vld1q_f32(a_scalars + idx_scalars);
267
- float32x4_t b_vec = vld1q_f32(b_scalars + idx_scalars);
268
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
269
- }
270
- simsimd_f32_t ab = vaddvq_f32(ab_vec);
271
- for (; idx_scalars < count_scalars; ++idx_scalars) ab += a_scalars[idx_scalars] * b_scalars[idx_scalars];
272
- *result = ab;
273
- }
274
-
275
- SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
276
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
277
- float32x4_t ab_real_vec = vdupq_n_f32(0);
278
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
279
- simsimd_size_t idx_pairs = 0;
280
- for (; idx_pairs + 4 <= count_pairs; idx_pairs += 4) {
281
- // Unpack the input arrays into real and imaginary parts:
282
- float32x4x2_t a_vec = vld2q_f32((simsimd_f32_t const *)(a_pairs + idx_pairs));
283
- float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b_pairs + idx_pairs));
284
- float32x4_t a_real_vec = a_vec.val[0];
285
- float32x4_t a_imag_vec = a_vec.val[1];
286
- float32x4_t b_real_vec = b_vec.val[0];
287
- float32x4_t b_imag_vec = b_vec.val[1];
288
-
289
- // Compute the dot product:
290
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
291
- ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
292
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
293
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
294
- }
295
-
296
- // Reduce horizontal sums:
297
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
298
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
299
-
300
- // Handle the tail:
301
- for (; idx_pairs != count_pairs; ++idx_pairs) {
302
- simsimd_f32c_t a_pair = a_pairs[idx_pairs], b_pair = b_pairs[idx_pairs];
303
- simsimd_f32_t ar = a_pair.real, ai = a_pair.imag, br = b_pair.real, bi = b_pair.imag;
304
- ab_real += ar * br - ai * bi;
305
- ab_imag += ar * bi + ai * br;
306
- }
307
- results[0] = ab_real;
308
- results[1] = ab_imag;
309
- }
310
-
311
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
312
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
313
- float32x4_t ab_real_vec = vdupq_n_f32(0);
314
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
315
- simsimd_size_t idx_pairs = 0;
316
- for (; idx_pairs + 4 <= count_pairs; idx_pairs += 4) {
317
- // Unpack the input arrays into real and imaginary parts:
318
- float32x4x2_t a_vec = vld2q_f32((simsimd_f32_t const *)(a_pairs + idx_pairs));
319
- float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b_pairs + idx_pairs));
320
- float32x4_t a_real_vec = a_vec.val[0];
321
- float32x4_t a_imag_vec = a_vec.val[1];
322
- float32x4_t b_real_vec = b_vec.val[0];
323
- float32x4_t b_imag_vec = b_vec.val[1];
324
-
325
- // Compute the dot product:
326
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
327
- ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
328
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
329
- ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
330
- }
331
-
332
- // Reduce horizontal sums:
333
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
334
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
335
-
336
- // Handle the tail:
337
- for (; idx_pairs != count_pairs; ++idx_pairs) {
338
- simsimd_f32c_t a_pair = a_pairs[idx_pairs], b_pair = b_pairs[idx_pairs];
339
- simsimd_f32_t ar = a_pair.real, ai = a_pair.imag, br = b_pair.real, bi = b_pair.imag;
340
- ab_real += ar * br + ai * bi;
341
- ab_imag += ar * bi - ai * br;
342
- }
343
- results[0] = ab_real;
344
- results[1] = ab_imag;
345
- }
346
-
347
- #pragma clang attribute pop
348
- #pragma GCC pop_options
349
- #endif // SIMSIMD_TARGET_NEON
350
-
351
- #if SIMSIMD_TARGET_NEON_I8
352
- #pragma GCC push_options
353
- #pragma GCC target("arch=armv8.2-a+dotprod")
354
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod"))), apply_to = function)
355
-
356
- SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const *a_scalars, simsimd_i8_t const *b_scalars,
357
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
358
-
359
- int32x4_t ab_vec = vdupq_n_s32(0);
360
- simsimd_size_t idx_scalars = 0;
361
-
362
- // If the 128-bit `vdot_s32` intrinsic is unavailable, we can use the 64-bit `vdot_s32`.
363
- // for (simsimd_size_t idx_scalars = 0; idx_scalars != n; idx_scalars += 8) {
364
- // int16x8_t a_vec = vmovl_s8(vld1_s8(a_scalars + idx_scalars));
365
- // int16x8_t b_vec = vmovl_s8(vld1_s8(b_scalars + idx_scalars));
366
- // int16x8_t ab_part_vec = vmulq_s16(a_vec, b_vec);
367
- // ab_vec = vaddq_s32(ab_vec, vaddq_s32(vmovl_s16(vget_high_s16(ab_part_vec)), //
368
- // vmovl_s16(vget_low_s16(ab_part_vec))));
369
- // }
370
- for (; idx_scalars + 16 <= count_scalars; idx_scalars += 16) {
371
- int8x16_t a_vec = vld1q_s8(a_scalars + idx_scalars);
372
- int8x16_t b_vec = vld1q_s8(b_scalars + idx_scalars);
373
- ab_vec = vdotq_s32(ab_vec, a_vec, b_vec);
374
- }
375
-
376
- // Take care of the tail:
377
- simsimd_i32_t ab = vaddvq_s32(ab_vec);
378
- for (; idx_scalars < count_scalars; ++idx_scalars) {
379
- simsimd_i32_t ai = a_scalars[idx_scalars], bi = b_scalars[idx_scalars];
380
- ab += ai * bi;
381
- }
382
-
383
- *result = ab;
384
- }
385
-
386
- SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const *a_scalars, simsimd_u8_t const *b_scalars,
387
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
388
-
389
- uint32x4_t ab_vec = vdupq_n_u32(0);
390
- simsimd_size_t idx_scalars = 0;
391
- for (; idx_scalars + 16 <= count_scalars; idx_scalars += 16) {
392
- uint8x16_t a_vec = vld1q_u8(a_scalars + idx_scalars);
393
- uint8x16_t b_vec = vld1q_u8(b_scalars + idx_scalars);
394
- ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
395
- }
396
-
397
- // Take care of the tail:
398
- simsimd_u32_t ab = vaddvq_u32(ab_vec);
399
- for (; idx_scalars < count_scalars; ++idx_scalars) {
400
- simsimd_u32_t ai = a_scalars[idx_scalars], bi = b_scalars[idx_scalars];
401
- ab += ai * bi;
402
- }
403
-
404
- *result = ab;
405
- }
406
-
407
- #pragma clang attribute pop
408
- #pragma GCC pop_options
409
- #endif // SIMSIMD_TARGET_NEON_I8
410
-
411
- #if SIMSIMD_TARGET_NEON_F16
412
- #pragma GCC push_options
413
- #pragma GCC target("arch=armv8.2-a+simd+fp16")
414
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
415
-
416
- SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const *x, simsimd_size_t n) {
417
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
418
- // function will run. It is extremely slow, so even for the tail, let's combine serial
419
- // loads and stores with vectorized math.
420
- union {
421
- float16x4_t vec;
422
- simsimd_f16_t scalars[4];
423
- } result;
424
- simsimd_size_t i = 0;
425
- for (; i < n; ++i) result.scalars[i] = x[i];
426
- for (; i < 4; ++i) result.scalars[i] = 0;
427
- return result.vec;
428
- }
429
-
430
- SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const *a_scalars, simsimd_f16_t const *b_scalars,
431
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
432
- float32x4_t a_vec, b_vec;
433
- float32x4_t ab_vec = vdupq_n_f32(0);
434
- simsimd_size_t i = 0;
435
-
436
- simsimd_dot_f16_neon_cycle:
437
- if (count_scalars < 4) {
438
- a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a_scalars, count_scalars));
439
- b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b_scalars, count_scalars));
440
- count_scalars = 0;
441
- }
442
- else {
443
- a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a_scalars));
444
- b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b_scalars));
445
- a_scalars += 4, b_scalars += 4, count_scalars -= 4;
446
- }
447
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
448
- if (count_scalars) goto simsimd_dot_f16_neon_cycle;
449
- *result = vaddvq_f32(ab_vec);
450
- }
451
-
452
- SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
453
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
454
-
455
- // A nicer approach is to use `f16` arithmetic for the dot product, but that requires
456
- // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
457
- // at once. That's how the original implementation worked, but compiling it was a nightmare :)
458
- float32x4_t ab_real_vec = vdupq_n_f32(0);
459
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
460
-
461
- while (count_pairs >= 4) {
462
- // Unpack the input arrays into real and imaginary parts.
463
- // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
464
- // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
465
- int16x4x2_t a_vec = vld2_s16((short *)a_pairs);
466
- int16x4x2_t b_vec = vld2_s16((short *)b_pairs);
467
- float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
468
- float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
469
- float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
470
- float32x4_t b_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[1]));
471
-
472
- // Compute the dot product:
473
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
474
- ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
475
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
476
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
477
-
478
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
479
- }
480
-
481
- // Reduce horizontal sums and aggregate with the tail:
482
- simsimd_dot_f16c_serial(a_pairs, b_pairs, count_pairs, results);
483
- results[0] += vaddvq_f32(ab_real_vec);
484
- results[1] += vaddvq_f32(ab_imag_vec);
485
- }
486
-
487
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
488
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
489
-
490
- // A nicer approach is to use `f16` arithmetic for the dot product, but that requires
491
- // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
492
- // at once. That's how the original implementation worked, but compiling it was a nightmare :)
493
- float32x4_t ab_real_vec = vdupq_n_f32(0);
494
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
495
-
496
- while (count_pairs >= 4) {
497
- // Unpack the input arrays into real and imaginary parts.
498
- // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
499
- // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
500
- int16x4x2_t a_vec = vld2_s16((short *)a_pairs);
501
- int16x4x2_t b_vec = vld2_s16((short *)b_pairs);
502
- float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
503
- float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
504
- float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
505
- float32x4_t b_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[1]));
506
-
507
- // Compute the dot product:
508
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
509
- ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
510
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
511
- ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
512
-
513
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
514
- }
515
-
516
- // Reduce horizontal sums and aggregate with the tail:
517
- simsimd_vdot_f16c_serial(a_pairs, b_pairs, count_pairs, results);
518
- results[0] += vaddvq_f32(ab_real_vec);
519
- results[1] += vaddvq_f32(ab_imag_vec);
520
- }
521
-
522
- #pragma clang attribute pop
523
- #pragma GCC pop_options
524
- #endif // SIMSIMD_TARGET_NEON_F16
525
-
526
- #if SIMSIMD_TARGET_NEON_BF16
527
- #pragma GCC push_options
528
- #pragma GCC target("arch=armv8.6-a+simd+bf16")
529
- #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
530
-
531
- SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const *x, simsimd_size_t n) {
532
- union {
533
- bfloat16x8_t vec;
534
- simsimd_bf16_t scalars[8];
535
- } result;
536
- simsimd_size_t i = 0;
537
- for (; i < n; ++i) result.scalars[i] = x[i];
538
- for (; i < 8; ++i) result.scalars[i] = 0;
539
- return result.vec;
540
- }
541
-
542
- SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const *a_scalars, simsimd_bf16_t const *b_scalars,
543
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
544
- bfloat16x8_t a_vec, b_vec;
545
- float32x4_t ab_vec = vdupq_n_f32(0);
546
-
547
- simsimd_dot_bf16_neon_cycle:
548
- if (count_scalars < 8) {
549
- a_vec = _simsimd_partial_load_bf16x8_neon(a_scalars, count_scalars);
550
- b_vec = _simsimd_partial_load_bf16x8_neon(b_scalars, count_scalars);
551
- count_scalars = 0;
552
- }
553
- else {
554
- a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a_scalars);
555
- b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b_scalars);
556
- a_scalars += 8, b_scalars += 8, count_scalars -= 8;
557
- }
558
- ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
559
- if (count_scalars) goto simsimd_dot_bf16_neon_cycle;
560
-
561
- *result = vaddvq_f32(ab_vec);
562
- }
563
-
564
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16c_t const *a_pairs, simsimd_bf16c_t const *b_pairs,
565
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
566
-
567
- // A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
568
- // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
569
- // at once. That's how the original implementation worked, but compiling it was a nightmare :)
570
- float32x4_t ab_real_vec = vdupq_n_f32(0);
571
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
572
-
573
- while (count_pairs >= 4) {
574
- // Unpack the input arrays into real and imaginary parts.
575
- // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
576
- // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
577
- int16x4x2_t a_vec = vld2_s16((short const *)a_pairs);
578
- int16x4x2_t b_vec = vld2_s16((short const *)b_pairs);
579
- float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
580
- float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
581
- float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
582
- float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
583
-
584
- // Compute the dot product:
585
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
586
- ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
587
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
588
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
589
-
590
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
591
- }
592
-
593
- // Reduce horizontal sums and aggregate with the tail:
594
- simsimd_dot_bf16c_serial(a_pairs, b_pairs, count_pairs, results);
595
- results[0] += vaddvq_f32(ab_real_vec);
596
- results[1] += vaddvq_f32(ab_imag_vec);
597
- }
598
-
599
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16c_t const *a_pairs, simsimd_bf16c_t const *b_pairs,
600
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
601
-
602
- // A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
603
- // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
604
- // at once. That's how the original implementation worked, but compiling it was a nightmare :)
605
- float32x4_t ab_real_vec = vdupq_n_f32(0);
606
- float32x4_t ab_imag_vec = vdupq_n_f32(0);
607
-
608
- while (count_pairs >= 4) {
609
- // Unpack the input arrays into real and imaginary parts.
610
- // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
611
- // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
612
- int16x4x2_t a_vec = vld2_s16((short const *)a_pairs);
613
- int16x4x2_t b_vec = vld2_s16((short const *)b_pairs);
614
- float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
615
- float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
616
- float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
617
- float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
618
-
619
- // Compute the dot product:
620
- ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
621
- ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
622
- ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
623
- ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
624
-
625
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
626
- }
627
-
628
- // Reduce horizontal sums and aggregate with the tail:
629
- simsimd_vdot_bf16c_serial(a_pairs, b_pairs, count_pairs, results);
630
- results[0] += vaddvq_f32(ab_real_vec);
631
- results[1] += vaddvq_f32(ab_imag_vec);
632
- }
633
-
634
- #pragma clang attribute pop
635
- #pragma GCC pop_options
636
- #endif // SIMSIMD_TARGET_NEON_BF16
637
-
638
- #if SIMSIMD_TARGET_SVE
639
-
640
- #pragma GCC push_options
641
- #pragma GCC target("arch=armv8.2-a+sve")
642
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
643
-
644
- SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const *a_scalars, simsimd_f32_t const *b_scalars,
645
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
646
- simsimd_size_t idx_scalars = 0;
647
- svfloat32_t ab_vec = svdup_f32(0.f);
648
- do {
649
- svbool_t pg_vec = svwhilelt_b32((unsigned int)idx_scalars, (unsigned int)count_scalars);
650
- svfloat32_t a_vec = svld1_f32(pg_vec, a_scalars + idx_scalars);
651
- svfloat32_t b_vec = svld1_f32(pg_vec, b_scalars + idx_scalars);
652
- ab_vec = svmla_f32_x(pg_vec, ab_vec, a_vec, b_vec);
653
- idx_scalars += svcntw();
654
- } while (idx_scalars < count_scalars);
655
- *result = svaddv_f32(svptrue_b32(), ab_vec);
656
- }
657
-
658
- SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
659
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
660
- simsimd_size_t idx_pairs = 0;
661
- svfloat32_t ab_real_vec = svdup_f32(0.f);
662
- svfloat32_t ab_imag_vec = svdup_f32(0.f);
663
- do {
664
- svbool_t pg_vec = svwhilelt_b32((unsigned int)idx_pairs, (unsigned int)count_pairs);
665
- svfloat32x2_t a_vec = svld2_f32(pg_vec, (simsimd_f32_t const *)(a_pairs + idx_pairs));
666
- svfloat32x2_t b_vec = svld2_f32(pg_vec, (simsimd_f32_t const *)(b_pairs + idx_pairs));
667
- svfloat32_t a_real_vec = svget2_f32(a_vec, 0);
668
- svfloat32_t a_imag_vec = svget2_f32(a_vec, 1);
669
- svfloat32_t b_real_vec = svget2_f32(b_vec, 0);
670
- svfloat32_t b_imag_vec = svget2_f32(b_vec, 1);
671
- ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
672
- ab_real_vec = svmls_f32_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
673
- ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
674
- ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
675
- idx_pairs += svcntw();
676
- } while (idx_pairs < count_pairs);
677
- results[0] = svaddv_f32(svptrue_b32(), ab_real_vec);
678
- results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
679
- }
680
-
681
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
682
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
683
- simsimd_size_t idx_pairs = 0;
684
- svfloat32_t ab_real_vec = svdup_f32(0.f);
685
- svfloat32_t ab_imag_vec = svdup_f32(0.f);
686
- do {
687
- svbool_t pg_vec = svwhilelt_b32((unsigned int)idx_pairs, (unsigned int)count_pairs);
688
- svfloat32x2_t a_vec = svld2_f32(pg_vec, (simsimd_f32_t const *)(a_pairs + idx_pairs));
689
- svfloat32x2_t b_vec = svld2_f32(pg_vec, (simsimd_f32_t const *)(b_pairs + idx_pairs));
690
- svfloat32_t a_real_vec = svget2_f32(a_vec, 0);
691
- svfloat32_t a_imag_vec = svget2_f32(a_vec, 1);
692
- svfloat32_t b_real_vec = svget2_f32(b_vec, 0);
693
- svfloat32_t b_imag_vec = svget2_f32(b_vec, 1);
694
- ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
695
- ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
696
- ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
697
- ab_imag_vec = svmls_f32_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
698
- idx_pairs += svcntw();
699
- } while (idx_pairs < count_pairs);
700
- results[0] = svaddv_f32(svptrue_b32(), ab_real_vec);
701
- results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
702
- }
703
-
704
- SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const *a_scalars, simsimd_f64_t const *b_scalars,
705
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
706
- simsimd_size_t idx_scalars = 0;
707
- svfloat64_t ab_vec = svdup_f64(0.);
708
- do {
709
- svbool_t pg_vec = svwhilelt_b64((unsigned int)idx_scalars, (unsigned int)count_scalars);
710
- svfloat64_t a_vec = svld1_f64(pg_vec, a_scalars + idx_scalars);
711
- svfloat64_t b_vec = svld1_f64(pg_vec, b_scalars + idx_scalars);
712
- ab_vec = svmla_f64_x(pg_vec, ab_vec, a_vec, b_vec);
713
- idx_scalars += svcntd();
714
- } while (idx_scalars < count_scalars);
715
- *result = svaddv_f64(svptrue_b32(), ab_vec);
716
- }
717
-
718
- SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64c_t const *a_pairs, simsimd_f64c_t const *b_pairs,
719
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
720
- simsimd_size_t idx_pairs = 0;
721
- svfloat64_t ab_real_vec = svdup_f64(0.);
722
- svfloat64_t ab_imag_vec = svdup_f64(0.);
723
- do {
724
- svbool_t pg_vec = svwhilelt_b64((unsigned int)idx_pairs, (unsigned int)count_pairs);
725
- svfloat64x2_t a_vec = svld2_f64(pg_vec, (simsimd_f64_t const *)(a_pairs + idx_pairs));
726
- svfloat64x2_t b_vec = svld2_f64(pg_vec, (simsimd_f64_t const *)(b_pairs + idx_pairs));
727
- svfloat64_t a_real_vec = svget2_f64(a_vec, 0);
728
- svfloat64_t a_imag_vec = svget2_f64(a_vec, 1);
729
- svfloat64_t b_real_vec = svget2_f64(b_vec, 0);
730
- svfloat64_t b_imag_vec = svget2_f64(b_vec, 1);
731
- ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
732
- ab_real_vec = svmls_f64_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
733
- ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
734
- ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
735
- idx_pairs += svcntd();
736
- } while (idx_pairs < count_pairs);
737
- results[0] = svaddv_f64(svptrue_b64(), ab_real_vec);
738
- results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
739
- }
740
-
741
- SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64c_t const *a_pairs, simsimd_f64c_t const *b_pairs,
742
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
743
- simsimd_size_t idx_pairs = 0;
744
- svfloat64_t ab_real_vec = svdup_f64(0.);
745
- svfloat64_t ab_imag_vec = svdup_f64(0.);
746
- do {
747
- svbool_t pg_vec = svwhilelt_b64((unsigned int)idx_pairs, (unsigned int)count_pairs);
748
- svfloat64x2_t a_vec = svld2_f64(pg_vec, (simsimd_f64_t const *)(a_pairs + idx_pairs));
749
- svfloat64x2_t b_vec = svld2_f64(pg_vec, (simsimd_f64_t const *)(b_pairs + idx_pairs));
750
- svfloat64_t a_real_vec = svget2_f64(a_vec, 0);
751
- svfloat64_t a_imag_vec = svget2_f64(a_vec, 1);
752
- svfloat64_t b_real_vec = svget2_f64(b_vec, 0);
753
- svfloat64_t b_imag_vec = svget2_f64(b_vec, 1);
754
- ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
755
- ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
756
- ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
757
- ab_imag_vec = svmls_f64_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
758
- idx_pairs += svcntd();
759
- } while (idx_pairs < count_pairs);
760
- results[0] = svaddv_f64(svptrue_b64(), ab_real_vec);
761
- results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
762
- }
763
-
764
- #pragma clang attribute pop
765
- #pragma GCC pop_options
766
-
767
- #pragma GCC push_options
768
- #pragma GCC target("arch=armv8.2-a+sve+fp16")
769
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
770
-
771
- SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const *a_scalars, simsimd_f16_t const *b_scalars,
772
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
773
- simsimd_size_t idx_scalars = 0;
774
- svfloat16_t ab_vec = svdup_f16(0);
775
- do {
776
- svbool_t pg_vec = svwhilelt_b16((unsigned int)idx_scalars, (unsigned int)count_scalars);
777
- svfloat16_t a_vec = svld1_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(a_scalars + idx_scalars));
778
- svfloat16_t b_vec = svld1_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(b_scalars + idx_scalars));
779
- ab_vec = svmla_f16_x(pg_vec, ab_vec, a_vec, b_vec);
780
- idx_scalars += svcnth();
781
- } while (idx_scalars < count_scalars);
782
- simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
783
- *result = ab;
784
- }
785
-
786
- SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
787
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
788
- simsimd_size_t idx_pairs = 0;
789
- svfloat16_t ab_real_vec = svdup_f16(0);
790
- svfloat16_t ab_imag_vec = svdup_f16(0);
791
- do {
792
- svbool_t pg_vec = svwhilelt_b32((unsigned int)idx_pairs, (unsigned int)count_pairs);
793
- svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(a_pairs + idx_pairs));
794
- svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(b_pairs + idx_pairs));
795
- svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
796
- svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
797
- svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
798
- svfloat16_t b_imag_vec = svget2_f16(b_vec, 1);
799
- ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
800
- ab_real_vec = svmls_f16_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
801
- ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
802
- ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
803
- idx_pairs += svcnth();
804
- } while (idx_pairs < count_pairs);
805
- results[0] = svaddv_f16(svptrue_b16(), ab_real_vec);
806
- results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
807
- }
808
-
809
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
810
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
811
- simsimd_size_t idx_pairs = 0;
812
- svfloat16_t ab_real_vec = svdup_f16(0);
813
- svfloat16_t ab_imag_vec = svdup_f16(0);
814
- do {
815
- svbool_t pg_vec = svwhilelt_b32((unsigned int)idx_pairs, (unsigned int)count_pairs);
816
- svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(a_pairs + idx_pairs));
817
- svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)(b_pairs + idx_pairs));
818
- svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
819
- svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
820
- svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
821
- svfloat16_t b_imag_vec = svget2_f16(b_vec, 1);
822
- ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
823
- ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
824
- ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
825
- ab_imag_vec = svmls_f16_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
826
- idx_pairs += svcnth();
827
- } while (idx_pairs < count_pairs);
828
- results[0] = svaddv_f16(svptrue_b16(), ab_real_vec);
829
- results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
830
- }
831
-
832
- #pragma clang attribute pop
833
- #pragma GCC pop_options
834
- #endif // SIMSIMD_TARGET_SVE
835
- #endif // _SIMSIMD_TARGET_ARM
836
-
837
- #if _SIMSIMD_TARGET_X86
838
- #if SIMSIMD_TARGET_HASWELL
839
- #pragma GCC push_options
840
- #pragma GCC target("avx2", "f16c", "fma")
841
- #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
842
-
843
- SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f64x4_haswell(__m256d vec) {
844
- // Reduce the double-precision vector to a scalar
845
- // Horizontal add the first and second double-precision values, and third and fourth
846
- __m128d vec_low = _mm256_castpd256_pd128(vec);
847
- __m128d vec_high = _mm256_extractf128_pd(vec, 1);
848
- __m128d vec128 = _mm_add_pd(vec_low, vec_high);
849
-
850
- // Horizontal add again to accumulate all four values into one
851
- vec128 = _mm_hadd_pd(vec128, vec128);
852
-
853
- // Convert the final sum to a scalar double-precision value and return
854
- return _mm_cvtsd_f64(vec128);
855
- }
856
-
857
- SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
858
- // Convert the lower and higher 128-bit lanes of the input vector to double precision
859
- __m128 low_f32 = _mm256_castps256_ps128(vec);
860
- __m128 high_f32 = _mm256_extractf128_ps(vec, 1);
861
-
862
- // Convert single-precision (float) vectors to double-precision (double) vectors
863
- __m256d low_f64 = _mm256_cvtps_pd(low_f32);
864
- __m256d high_f64 = _mm256_cvtps_pd(high_f32);
865
-
866
- // Perform the addition in double-precision
867
- __m256d sum = _mm256_add_pd(low_f64, high_f64);
868
- return _simsimd_reduce_f64x4_haswell(sum);
869
- }
870
-
871
- SIMSIMD_INTERNAL simsimd_i32_t _simsimd_reduce_i32x8_haswell(__m256i vec) {
872
- __m128i low = _mm256_castsi256_si128(vec);
873
- __m128i high = _mm256_extracti128_si256(vec, 1);
874
- __m128i sum = _mm_add_epi32(low, high);
875
- sum = _mm_hadd_epi32(sum, sum);
876
- sum = _mm_hadd_epi32(sum, sum);
877
- return _mm_cvtsi128_si32(sum);
878
- }
879
-
880
- SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const *a_scalars, simsimd_f32_t const *b_scalars,
881
- simsimd_size_t count_scalars, simsimd_distance_t *results) {
882
-
883
- __m256 ab_vec = _mm256_setzero_ps();
884
- simsimd_size_t idx_scalars = 0;
885
- for (; idx_scalars + 8 <= count_scalars; idx_scalars += 8) {
886
- __m256 a_vec = _mm256_loadu_ps(a_scalars + idx_scalars);
887
- __m256 b_vec = _mm256_loadu_ps(b_scalars + idx_scalars);
888
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
889
- }
890
- simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
891
- for (; idx_scalars < count_scalars; ++idx_scalars) ab += a_scalars[idx_scalars] * b_scalars[idx_scalars];
892
- *results = ab;
893
- }
894
-
895
- SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
896
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
897
-
898
- // The naive approach would be to use FMA and FMS instructions on different parts of the vectors.
899
- // Prior to that we would need to shuffle the input vectors to separate real and imaginary parts.
900
- // Both operations are quite expensive, and the resulting kernel would run at 2.5 GB/s.
901
- // __m128 ab_real_vec = _mm_setzero_ps();
902
- // __m128 ab_imag_vec = _mm_setzero_ps();
903
- // __m256i permute_vec = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);
904
- // simsimd_size_t idx_pairs = 0;
905
- // for (; idx_pairs + 4 <= count_pairs; idx_pairs += 4) {
906
- // __m256 a_vec = _mm256_loadu_ps((simsimd_f32_t const *)(a_pairs + idx_pairs));
907
- // __m256 b_vec = _mm256_loadu_ps((simsimd_f32_t const *)(b_pairs + idx_pairs));
908
- // __m256 a_shuffled = _mm256_permutevar8x32_ps(a_vec, permute_vec);
909
- // __m256 b_shuffled = _mm256_permutevar8x32_ps(b_vec, permute_vec);
910
- // __m128 a_real_vec = _mm256_extractf128_ps(a_shuffled, 0);
911
- // __m128 a_imag_vec = _mm256_extractf128_ps(a_shuffled, 1);
912
- // __m128 b_real_vec = _mm256_extractf128_ps(b_shuffled, 0);
913
- // __m128 b_imag_vec = _mm256_extractf128_ps(b_shuffled, 1);
914
- // ab_real_vec = _mm_fmadd_ps(a_real_vec, b_real_vec, ab_real_vec);
915
- // ab_real_vec = _mm_fnmadd_ps(a_imag_vec, b_imag_vec, ab_real_vec);
916
- // ab_imag_vec = _mm_fmadd_ps(a_real_vec, b_imag_vec, ab_imag_vec);
917
- // ab_imag_vec = _mm_fmadd_ps(a_imag_vec, b_real_vec, ab_imag_vec);
918
- // }
919
- //
920
- // Instead, we take into account, that FMS is the same as FMA with a negative multiplier.
921
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
922
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
923
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
924
- // one of the vectors. Moreover, `XOR` can be placed after the primary loop.
925
- // Both operations are quite cheap, and the throughput doubles from 2.5 GB/s to 5 GB/s.
926
- __m256 ab_real_vec = _mm256_setzero_ps();
927
- __m256 ab_imag_vec = _mm256_setzero_ps();
928
- __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
929
- __m256i swap_adjacent_vec = _mm256_set_epi8( //
930
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
931
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
932
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
933
- 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
934
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
935
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
936
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
937
- 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
938
- );
939
-
940
- simsimd_size_t idx_pairs = 0;
941
- for (; idx_pairs + 4 <= count_pairs; idx_pairs += 4) {
942
- __m256 a_vec = _mm256_loadu_ps((simsimd_f32_t const *)(a_pairs + idx_pairs));
943
- __m256 b_vec = _mm256_loadu_ps((simsimd_f32_t const *)(b_pairs + idx_pairs));
944
- __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
945
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
946
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
947
- }
948
-
949
- // Flip the sign bit in every second scalar before accumulation:
950
- ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
951
-
952
- // Reduce horizontal sums:
953
- simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
954
- simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
955
-
956
- // Handle the tail:
957
- for (; idx_pairs != count_pairs; ++idx_pairs) {
958
- simsimd_f32c_t a_pair = a_pairs[idx_pairs], b_pair = b_pairs[idx_pairs];
959
- simsimd_f32_t ar = a_pair.real, ai = a_pair.imag, br = b_pair.real, bi = b_pair.imag;
960
- ab_real += ar * br - ai * bi;
961
- ab_imag += ar * bi + ai * br;
962
- }
963
- results[0] = ab_real;
964
- results[1] = ab_imag;
965
- }
966
-
967
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
968
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
969
-
970
- __m256 ab_real_vec = _mm256_setzero_ps();
971
- __m256 ab_imag_vec = _mm256_setzero_ps();
972
- __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
973
- __m256i swap_adjacent_vec = _mm256_set_epi8( //
974
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
975
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
976
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
977
- 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
978
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
979
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
980
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
981
- 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
982
- );
983
-
984
- simsimd_size_t idx_pairs = 0;
985
- for (; idx_pairs + 4 <= count_pairs; idx_pairs += 4) {
986
- __m256 a_vec = _mm256_loadu_ps((simsimd_f32_t const *)(a_pairs + idx_pairs));
987
- __m256 b_vec = _mm256_loadu_ps((simsimd_f32_t const *)(b_pairs + idx_pairs));
988
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
989
- b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
990
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
991
- }
992
-
993
- // Flip the sign bit in every second scalar before accumulation:
994
- ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
995
-
996
- // Reduce horizontal sums:
997
- simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
998
- simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
999
-
1000
- // Handle the tail:
1001
- for (; idx_pairs != count_pairs; ++idx_pairs) {
1002
- simsimd_f32c_t a_pair = a_pairs[idx_pairs], b_pair = b_pairs[idx_pairs];
1003
- simsimd_f32_t ar = a_pair.real, ai = a_pair.imag, br = b_pair.real, bi = b_pair.imag;
1004
- ab_real += ar * br + ai * bi;
1005
- ab_imag += ar * bi - ai * br;
1006
- }
1007
- results[0] = ab_real;
1008
- results[1] = ab_imag;
1009
- }
1010
-
1011
- SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const *a, simsimd_size_t n) {
1012
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
1013
- // function will run. It is extremely slow, so even for the tail, let's combine serial
1014
- // loads and stores with vectorized math.
1015
- union {
1016
- __m128i vec;
1017
- simsimd_f16_t scalars[8];
1018
- } result;
1019
- simsimd_size_t i = 0;
1020
- for (; i < n; ++i) result.scalars[i] = a[i];
1021
- for (; i < 8; ++i) result.scalars[i] = 0;
1022
- return _mm256_cvtph_ps(result.vec);
1023
- }
1024
-
1025
- SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const *a_scalars, simsimd_f16_t const *b_scalars,
1026
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1027
- __m256 a_vec, b_vec;
1028
- __m256 ab_vec = _mm256_setzero_ps();
1029
-
1030
- simsimd_dot_f16_haswell_cycle:
1031
- if (count_scalars < 8) {
1032
- a_vec = _simsimd_partial_load_f16x8_haswell(a_scalars, count_scalars);
1033
- b_vec = _simsimd_partial_load_f16x8_haswell(b_scalars, count_scalars);
1034
- count_scalars = 0;
1035
- }
1036
- else {
1037
- a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a_scalars));
1038
- b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b_scalars));
1039
- count_scalars -= 8, a_scalars += 8, b_scalars += 8;
1040
- }
1041
- // We can silence the NaNs using blends:
1042
- //
1043
- // __m256 a_is_nan = _mm256_cmp_ps(a_vec, a_vec, _CMP_UNORD_Q);
1044
- // __m256 b_is_nan = _mm256_cmp_ps(b_vec, b_vec, _CMP_UNORD_Q);
1045
- // ab_vec = _mm256_blendv_ps(_mm256_fmadd_ps(a_vec, b_vec, ab_vec), ab_vec, _mm256_or_ps(a_is_nan, b_is_nan));
1046
- //
1047
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1048
- if (count_scalars) goto simsimd_dot_f16_haswell_cycle;
1049
-
1050
- *result = _simsimd_reduce_f32x8_haswell(ab_vec);
1051
- }
1052
-
1053
- SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
1054
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1055
-
1056
- // Ideally the implementation would load 256 bits worth of vector data at a time,
1057
- // shuffle those within a register, split in halfs, and only then upcast.
1058
- // That way, we are stepping through 32x 16-bit vector components at a time, or 16 dimensions.
1059
- // Sadly, shuffling 16-bit entries in a YMM register is hard to implement efficiently.
1060
- //
1061
- // Simpler approach is to load 128 bits at a time, upcast, and then shuffle.
1062
- // This mostly replicates the `simsimd_dot_f32c_haswell`.
1063
- __m256 ab_real_vec = _mm256_setzero_ps();
1064
- __m256 ab_imag_vec = _mm256_setzero_ps();
1065
- __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
1066
- __m256i swap_adjacent_vec = _mm256_set_epi8( //
1067
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
1068
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
1069
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1070
- 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
1071
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
1072
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
1073
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1074
- 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
1075
- );
1076
-
1077
- while (count_pairs >= 4) {
1078
- __m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a_pairs));
1079
- __m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b_pairs));
1080
- __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1081
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
1082
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
1083
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
1084
- }
1085
-
1086
- // Flip the sign bit in every second scalar before accumulation:
1087
- ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
1088
-
1089
- // Reduce horizontal sums and aggregate with the tail:
1090
- simsimd_dot_f16c_serial(a_pairs, b_pairs, count_pairs, results);
1091
- results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
1092
- results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
1093
- }
1094
-
1095
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
1096
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1097
-
1098
- // Ideally the implementation would load 256 bits worth of vector data at a time,
1099
- // shuffle those within a register, split in halfs, and only then upcast.
1100
- // That way, we are stepping through 32x 16-bit vector components at a time, or 16 dimensions.
1101
- // Sadly, shuffling 16-bit entries in a YMM register is hard to implement efficiently.
1102
- //
1103
- // Simpler approach is to load 128 bits at a time, upcast, and then shuffle.
1104
- // This mostly replicates the `simsimd_vdot_f32c_haswell`.
1105
- __m256 ab_real_vec = _mm256_setzero_ps();
1106
- __m256 ab_imag_vec = _mm256_setzero_ps();
1107
- __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
1108
- __m256i swap_adjacent_vec = _mm256_set_epi8( //
1109
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
1110
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
1111
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1112
- 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
1113
- 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
1114
- 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
1115
- 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1116
- 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
1117
- );
1118
-
1119
- while (count_pairs >= 4) {
1120
- __m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a_pairs));
1121
- __m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b_pairs));
1122
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
1123
- b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1124
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1125
- count_pairs -= 4, a_pairs += 4, b_pairs += 4;
1126
- }
1127
-
1128
- // Flip the sign bit in every second scalar before accumulation:
1129
- ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
1130
-
1131
- // Reduce horizontal sums and aggregate with the tail:
1132
- simsimd_dot_f16c_serial(a_pairs, b_pairs, count_pairs, results);
1133
- results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
1134
- results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
1135
- }
1136
-
1137
- SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const *a_scalars, simsimd_i8_t const *b_scalars,
1138
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1139
-
1140
- __m256i ab_i32_low_vec = _mm256_setzero_si256();
1141
- __m256i ab_i32_high_vec = _mm256_setzero_si256();
1142
-
1143
- // AVX2 has no instructions for 8-bit signed integer dot-products,
1144
- // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1145
- // So we need to normalize the first vector to its absolute value,
1146
- // and shift the product sign into the second vector.
1147
- //
1148
- // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1149
- // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1150
- // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1151
- //
1152
- // The problem with this approach, however, is the `-128` value in the second vector.
1153
- // Flipping it's sign will do nothing, and the result will be incorrect.
1154
- // This can easily lead to noticeable numerical errors in the final result.
1155
- simsimd_size_t idx_scalars = 0;
1156
- for (; idx_scalars + 32 <= count_scalars; idx_scalars += 32) {
1157
- __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a_scalars + idx_scalars));
1158
- __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b_scalars + idx_scalars));
1159
-
1160
- // Upcast `int8` to `int16`
1161
- __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
1162
- __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1163
- __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
1164
- __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1165
-
1166
- // Multiply and accumulate at `int16` level, accumulate at `int32` level
1167
- ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1168
- ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1169
- }
1170
-
1171
- // Horizontal sum across the 256-bit register
1172
- int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1173
-
1174
- // Take care of the tail:
1175
- for (; idx_scalars < count_scalars; ++idx_scalars) ab += (int)(a_scalars[idx_scalars]) * b_scalars[idx_scalars];
1176
- *result = ab;
1177
- }
1178
-
1179
- SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const *a_scalars, simsimd_u8_t const *b_scalars,
1180
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1181
-
1182
- __m256i ab_i32_low_vec = _mm256_setzero_si256();
1183
- __m256i ab_i32_high_vec = _mm256_setzero_si256();
1184
- __m256i const zeros_vec = _mm256_setzero_si256();
1185
-
1186
- // AVX2 has no instructions for unsigned 8-bit integer dot-products,
1187
- // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1188
- simsimd_size_t idx_scalars = 0;
1189
- for (; idx_scalars + 32 <= count_scalars; idx_scalars += 32) {
1190
- __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a_scalars + idx_scalars));
1191
- __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b_scalars + idx_scalars));
1192
-
1193
- // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1194
- // instructions instead of extracts, as they are much faster and more efficient.
1195
- __m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
1196
- __m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
1197
- __m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
1198
- __m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
1199
-
1200
- // Multiply and accumulate at `int16` level, accumulate at `int32` level
1201
- ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1202
- ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1203
- }
1204
-
1205
- // Horizontal sum across the 256-bit register
1206
- int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1207
-
1208
- // Take care of the tail:
1209
- for (; idx_scalars < count_scalars; ++idx_scalars) ab += (int)(a_scalars[idx_scalars]) * b_scalars[idx_scalars];
1210
- *result = ab;
1211
- }
1212
-
1213
- SIMSIMD_INTERNAL __m256 _simsimd_bf16x8_to_f32x8_haswell(__m128i x) {
1214
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
1215
- return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16));
1216
- }
1217
-
1218
- SIMSIMD_INTERNAL __m128i _simsimd_f32x8_to_bf16x8_haswell(__m256 x) {
1219
- // Pack the 32-bit integers into 16-bit integers.
1220
- // This is less trivial than unpacking: https://stackoverflow.com/a/77781241/2766161
1221
- // The best approach is to shuffle within lanes first: https://stackoverflow.com/a/49723746/2766161
1222
- // Our shuffling mask will drop the low 2-bytes from every 4-byte word.
1223
- __m256i trunc_elements = _mm256_shuffle_epi8( //
1224
- _mm256_castps_si256(x), //
1225
- _mm256_set_epi8( //
1226
- -1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2, //
1227
- -1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2 //
1228
- ));
1229
- __m256i ordered = _mm256_permute4x64_epi64(trunc_elements, 0x58);
1230
- __m128i result = _mm256_castsi256_si128(ordered);
1231
- return result;
1232
- }
1233
-
1234
- SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t const *a, simsimd_size_t n) {
1235
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_bf16_to_f32`
1236
- // function will run. It is extremely slow, so even for the tail, let's combine serial
1237
- // loads and stores with vectorized math.
1238
- union {
1239
- __m128i vec;
1240
- simsimd_bf16_t scalars[8];
1241
- } result;
1242
- simsimd_size_t i = 0;
1243
- for (; i < n; ++i) result.scalars[i] = a[i];
1244
- for (; i < 8; ++i) result.scalars[i] = 0;
1245
- return result.vec;
1246
- }
1247
-
1248
- SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const *a_scalars, simsimd_bf16_t const *b_scalars,
1249
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1250
- __m128i a_vec, b_vec;
1251
- __m256 ab_vec = _mm256_setzero_ps();
1252
-
1253
- simsimd_dot_bf16_haswell_cycle:
1254
- if (count_scalars < 8) {
1255
- a_vec = _simsimd_partial_load_bf16x8_haswell(a_scalars, count_scalars);
1256
- b_vec = _simsimd_partial_load_bf16x8_haswell(b_scalars, count_scalars);
1257
- count_scalars = 0;
1258
- }
1259
- else {
1260
- a_vec = _mm_lddqu_si128((__m128i const *)a_scalars);
1261
- b_vec = _mm_lddqu_si128((__m128i const *)b_scalars);
1262
- a_scalars += 8, b_scalars += 8, count_scalars -= 8;
1263
- }
1264
- ab_vec = _mm256_fmadd_ps(_simsimd_bf16x8_to_f32x8_haswell(a_vec), _simsimd_bf16x8_to_f32x8_haswell(b_vec), ab_vec);
1265
- if (count_scalars) goto simsimd_dot_bf16_haswell_cycle;
1266
-
1267
- *result = _simsimd_reduce_f32x8_haswell(ab_vec);
1268
- }
1269
-
1270
- #pragma clang attribute pop
1271
- #pragma GCC pop_options
1272
- #endif // SIMSIMD_TARGET_HASWELL
1273
-
1274
- #if SIMSIMD_TARGET_SKYLAKE
1275
- #pragma GCC push_options
1276
- #pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
1277
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
1278
-
1279
- SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x16_skylake(__m512 a) {
1280
- __m512 x = _mm512_add_ps(a, _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 0, 3, 2)));
1281
- __m128 r = _mm512_castps512_ps128(_mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, _MM_SHUFFLE(0, 0, 0, 1))));
1282
- r = _mm_hadd_ps(r, r);
1283
- return _mm_cvtss_f32(_mm_hadd_ps(r, r));
1284
- }
1285
-
1286
- SIMSIMD_INTERNAL __m512 _simsimd_bf16x16_to_f32x16_skylake(__m256i a) {
1287
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
1288
- return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
1289
- }
1290
-
1291
- SIMSIMD_INTERNAL __m256i _simsimd_f32x16_to_bf16x16_skylake(__m512 a) {
1292
- // Add 2^15 and right shift 16 to do round-nearest
1293
- __m512i x = _mm512_srli_epi32(_mm512_add_epi32(_mm512_castps_si512(a), _mm512_set1_epi32(1 << 15)), 16);
1294
- return _mm512_cvtepi32_epi16(x);
1295
- }
1296
-
1297
- SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const *a_scalars, simsimd_f32_t const *b_scalars,
1298
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1299
- __m512 a_vec, b_vec;
1300
- __m512 ab_vec = _mm512_setzero();
1301
-
1302
- simsimd_dot_f32_skylake_cycle:
1303
- if (count_scalars < 16) {
1304
- __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, count_scalars);
1305
- a_vec = _mm512_maskz_loadu_ps(mask, a_scalars);
1306
- b_vec = _mm512_maskz_loadu_ps(mask, b_scalars);
1307
- count_scalars = 0;
1308
- }
1309
- else {
1310
- a_vec = _mm512_loadu_ps(a_scalars);
1311
- b_vec = _mm512_loadu_ps(b_scalars);
1312
- a_scalars += 16, b_scalars += 16, count_scalars -= 16;
1313
- }
1314
- ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
1315
- if (count_scalars) goto simsimd_dot_f32_skylake_cycle;
1316
-
1317
- *result = _simsimd_reduce_f32x16_skylake(ab_vec);
1318
- }
1319
-
1320
- SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const *a_scalars, simsimd_f64_t const *b_scalars,
1321
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1322
- __m512d a_vec, b_vec;
1323
- __m512d ab_vec = _mm512_setzero_pd();
1324
-
1325
- simsimd_dot_f64_skylake_cycle:
1326
- if (count_scalars < 8) {
1327
- __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, count_scalars);
1328
- a_vec = _mm512_maskz_loadu_pd(mask, a_scalars);
1329
- b_vec = _mm512_maskz_loadu_pd(mask, b_scalars);
1330
- count_scalars = 0;
1331
- }
1332
- else {
1333
- a_vec = _mm512_loadu_pd(a_scalars);
1334
- b_vec = _mm512_loadu_pd(b_scalars);
1335
- a_scalars += 8, b_scalars += 8, count_scalars -= 8;
1336
- }
1337
- ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
1338
- if (count_scalars) goto simsimd_dot_f64_skylake_cycle;
1339
-
1340
- *result = _mm512_reduce_add_pd(ab_vec);
1341
- }
1342
-
1343
- SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
1344
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1345
- __m512 a_vec, b_vec;
1346
- __m512 ab_real_vec = _mm512_setzero();
1347
- __m512 ab_imag_vec = _mm512_setzero();
1348
-
1349
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1350
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1351
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1352
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1353
- // one of the vectors.
1354
- __m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
1355
- simsimd_dot_f32c_skylake_cycle:
1356
- if (count_pairs < 8) {
1357
- __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1358
- a_vec = _mm512_maskz_loadu_ps(mask, a_pairs);
1359
- b_vec = _mm512_maskz_loadu_ps(mask, b_pairs);
1360
- count_pairs = 0;
1361
- }
1362
- else {
1363
- a_vec = _mm512_loadu_ps(a_pairs);
1364
- b_vec = _mm512_loadu_ps(b_pairs);
1365
- a_pairs += 8, b_pairs += 8, count_pairs -= 8;
1366
- }
1367
- ab_real_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_real_vec);
1368
- b_vec = _mm512_permute_ps(b_vec, 0xB1); //? Swap adjacent entries within each pair
1369
- ab_imag_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_imag_vec);
1370
- if (count_pairs) goto simsimd_dot_f32c_skylake_cycle;
1371
-
1372
- // Flip the sign bit in every second scalar before accumulation:
1373
- ab_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_real_vec), sign_flip_vec));
1374
-
1375
- // Reduce horizontal sums:
1376
- results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1377
- results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1378
- }
1379
-
1380
- SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32c_t const *a_pairs, simsimd_f32c_t const *b_pairs,
1381
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1382
- __m512 a_vec, b_vec;
1383
- __m512 ab_real_vec = _mm512_setzero();
1384
- __m512 ab_imag_vec = _mm512_setzero();
1385
-
1386
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1387
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1388
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1389
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1390
- // one of the vectors.
1391
- __m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
1392
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1393
- 59, 58, 57, 56, 63, 62, 61, 60, 51, 50, 49, 48, 55, 54, 53, 52, // 4th 128-bit lane
1394
- 43, 42, 41, 40, 47, 46, 45, 44, 35, 34, 33, 32, 39, 38, 37, 36, // 3rd 128-bit lane
1395
- 27, 26, 25, 24, 31, 30, 29, 28, 19, 18, 17, 16, 23, 22, 21, 20, // 2nd 128-bit lane
1396
- 11, 10, 9, 8, 15, 14, 13, 12, 3, 2, 1, 0, 7, 6, 5, 4 // 1st 128-bit lane
1397
- );
1398
- simsimd_vdot_f32c_skylake_cycle:
1399
- if (count_pairs < 8) {
1400
- __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1401
- a_vec = _mm512_maskz_loadu_ps(mask, (simsimd_f32_t const *)a_pairs);
1402
- b_vec = _mm512_maskz_loadu_ps(mask, (simsimd_f32_t const *)b_pairs);
1403
- count_pairs = 0;
1404
- }
1405
- else {
1406
- a_vec = _mm512_loadu_ps((simsimd_f32_t const *)a_pairs);
1407
- b_vec = _mm512_loadu_ps((simsimd_f32_t const *)b_pairs);
1408
- a_pairs += 8, b_pairs += 8, count_pairs -= 8;
1409
- }
1410
- ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
1411
- b_vec = _mm512_permute_ps(b_vec, 0xB1); //? Swap adjacent entries within each pair
1412
- ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1413
- if (count_pairs) goto simsimd_vdot_f32c_skylake_cycle;
1414
-
1415
- // Flip the sign bit in every second scalar before accumulation:
1416
- ab_imag_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_imag_vec), sign_flip_vec));
1417
-
1418
- // Reduce horizontal sums:
1419
- results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1420
- results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1421
- }
1422
-
1423
- SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64c_t const *a_pairs, simsimd_f64c_t const *b_pairs,
1424
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1425
- __m512d a_vec, b_vec;
1426
- __m512d ab_real_vec = _mm512_setzero_pd();
1427
- __m512d ab_imag_vec = _mm512_setzero_pd();
1428
-
1429
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1430
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1431
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1432
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1433
- // one of the vectors.
1434
- __m512i const sign_flip_vec = _mm512_set_epi64( //
1435
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1436
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1437
- );
1438
- simsimd_dot_f64c_skylake_cycle:
1439
- if (count_pairs < 4) {
1440
- __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1441
- a_vec = _mm512_maskz_loadu_pd(mask, a_pairs);
1442
- b_vec = _mm512_maskz_loadu_pd(mask, b_pairs);
1443
- count_pairs = 0;
1444
- }
1445
- else {
1446
- a_vec = _mm512_loadu_pd(a_pairs);
1447
- b_vec = _mm512_loadu_pd(b_pairs);
1448
- a_pairs += 4, b_pairs += 4, count_pairs -= 4;
1449
- }
1450
- ab_real_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_real_vec);
1451
- b_vec = _mm512_permute_pd(b_vec, 0x55); //? Same as 0b01010101.
1452
- ab_imag_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_imag_vec);
1453
- if (count_pairs) goto simsimd_dot_f64c_skylake_cycle;
1454
-
1455
- // Flip the sign bit in every second scalar before accumulation:
1456
- ab_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_real_vec), sign_flip_vec));
1457
-
1458
- // Reduce horizontal sums:
1459
- results[0] = _mm512_reduce_add_pd(ab_real_vec);
1460
- results[1] = _mm512_reduce_add_pd(ab_imag_vec);
1461
- }
1462
-
1463
- SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64c_t const *a_pairs, simsimd_f64c_t const *b_pairs,
1464
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1465
- __m512d a_vec, b_vec;
1466
- __m512d ab_real_vec = _mm512_setzero_pd();
1467
- __m512d ab_imag_vec = _mm512_setzero_pd();
1468
-
1469
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1470
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1471
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1472
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1473
- // one of the vectors.
1474
- __m512i const sign_flip_vec = _mm512_set_epi64( //
1475
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1476
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1477
- );
1478
- simsimd_vdot_f64c_skylake_cycle:
1479
- if (count_pairs < 4) {
1480
- __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1481
- a_vec = _mm512_maskz_loadu_pd(mask, (simsimd_f32_t const *)a_pairs);
1482
- b_vec = _mm512_maskz_loadu_pd(mask, (simsimd_f32_t const *)b_pairs);
1483
- count_pairs = 0;
1484
- }
1485
- else {
1486
- a_vec = _mm512_loadu_pd((simsimd_f32_t const *)a_pairs);
1487
- b_vec = _mm512_loadu_pd((simsimd_f32_t const *)b_pairs);
1488
- a_pairs += 4, b_pairs += 4, count_pairs -= 4;
1489
- }
1490
- ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
1491
- b_vec = _mm512_permute_pd(b_vec, 0x55); //? Same as 0b01010101.
1492
- ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
1493
- if (count_pairs) goto simsimd_vdot_f64c_skylake_cycle;
1494
-
1495
- // Flip the sign bit in every second scalar before accumulation:
1496
- ab_imag_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_imag_vec), sign_flip_vec));
1497
-
1498
- // Reduce horizontal sums:
1499
- results[0] = _mm512_reduce_add_pd(ab_real_vec);
1500
- results[1] = _mm512_reduce_add_pd(ab_imag_vec);
1501
- }
1502
-
1503
- #pragma clang attribute pop
1504
- #pragma GCC pop_options
1505
- #endif // SIMSIMD_TARGET_SKYLAKE
1506
-
1507
- #if SIMSIMD_TARGET_GENOA
1508
- #pragma GCC push_options
1509
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1510
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1511
- apply_to = function)
1512
-
1513
- SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const *a_scalars, simsimd_bf16_t const *b_scalars,
1514
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1515
- __m512i a_i16_vec, b_i16_vec;
1516
- __m512 ab_vec = _mm512_setzero_ps();
1517
-
1518
- simsimd_dot_bf16_genoa_cycle:
1519
- if (count_scalars < 32) {
1520
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
1521
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a_scalars);
1522
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b_scalars);
1523
- count_scalars = 0;
1524
- }
1525
- else {
1526
- a_i16_vec = _mm512_loadu_epi16(a_scalars);
1527
- b_i16_vec = _mm512_loadu_epi16(b_scalars);
1528
- a_scalars += 32, b_scalars += 32, count_scalars -= 32;
1529
- }
1530
- ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
1531
- if (count_scalars) goto simsimd_dot_bf16_genoa_cycle;
1532
-
1533
- *result = _simsimd_reduce_f32x16_skylake(ab_vec);
1534
- }
1535
-
1536
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16c_t const *a_pairs, simsimd_bf16c_t const *b_pairs,
1537
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1538
- __m512i a_vec, b_vec;
1539
- __m512 ab_real_vec = _mm512_setzero_ps();
1540
- __m512 ab_imag_vec = _mm512_setzero_ps();
1541
-
1542
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1543
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1544
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1545
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1546
- // one of the vectors.
1547
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1548
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1549
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1550
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1551
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1552
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1553
- );
1554
-
1555
- simsimd_dot_bf16c_genoa_cycle:
1556
- if (count_pairs < 16) {
1557
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1558
- a_vec = _mm512_maskz_loadu_epi16(mask, (simsimd_i16_t const *)a_pairs);
1559
- b_vec = _mm512_maskz_loadu_epi16(mask, (simsimd_i16_t const *)b_pairs);
1560
- count_pairs = 0;
1561
- }
1562
- else {
1563
- a_vec = _mm512_loadu_epi16((simsimd_i16_t const *)a_pairs);
1564
- b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)b_pairs);
1565
- a_pairs += 16, b_pairs += 16, count_pairs -= 16;
1566
- }
1567
- ab_real_vec = _mm512_dpbf16_ps(ab_real_vec, (__m512bh)(_mm512_xor_si512(b_vec, sign_flip_vec)), (__m512bh)(a_vec));
1568
- ab_imag_vec =
1569
- _mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), (__m512bh)(a_vec));
1570
- if (count_pairs) goto simsimd_dot_bf16c_genoa_cycle;
1571
-
1572
- // Reduce horizontal sums:
1573
- results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1574
- results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1575
- }
1576
-
1577
- SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16c_t const *a_pairs, simsimd_bf16c_t const *b_pairs,
1578
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1579
- __m512i a_vec, b_vec;
1580
- __m512 ab_real_vec = _mm512_setzero_ps();
1581
- __m512 ab_imag_vec = _mm512_setzero_ps();
1582
-
1583
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1584
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1585
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1586
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1587
- // one of the vectors.
1588
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1589
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1590
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1591
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1592
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1593
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1594
- );
1595
-
1596
- simsimd_dot_bf16c_genoa_cycle:
1597
- if (count_pairs < 16) {
1598
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1599
- a_vec = _mm512_maskz_loadu_epi16(mask, (simsimd_i16_t const *)a_pairs);
1600
- b_vec = _mm512_maskz_loadu_epi16(mask, (simsimd_i16_t const *)b_pairs);
1601
- count_pairs = 0;
1602
- }
1603
- else {
1604
- a_vec = _mm512_loadu_epi16((simsimd_i16_t const *)a_pairs);
1605
- b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)b_pairs);
1606
- a_pairs += 16, b_pairs += 16, count_pairs -= 16;
1607
- }
1608
- ab_real_vec = _mm512_dpbf16_ps(ab_real_vec, (__m512bh)(a_vec), (__m512bh)(b_vec));
1609
- a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
1610
- b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
1611
- ab_imag_vec = _mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(a_vec), (__m512bh)(b_vec));
1612
- if (count_pairs) goto simsimd_dot_bf16c_genoa_cycle;
1613
-
1614
- // Reduce horizontal sums:
1615
- results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1616
- results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1617
- }
1618
-
1619
- #pragma clang attribute pop
1620
- #pragma GCC pop_options
1621
- #endif // SIMSIMD_TARGET_GENOA
1622
-
1623
- #if SIMSIMD_TARGET_SAPPHIRE
1624
- #pragma GCC push_options
1625
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1626
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
1627
- apply_to = function)
1628
-
1629
- SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const *a_scalars, simsimd_f16_t const *b_scalars,
1630
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1631
- __m512i a_i16_vec, b_i16_vec;
1632
- __m512h ab_vec = _mm512_setzero_ph();
1633
-
1634
- simsimd_dot_f16_sapphire_cycle:
1635
- if (count_scalars < 32) {
1636
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
1637
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a_scalars);
1638
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b_scalars);
1639
- count_scalars = 0;
1640
- }
1641
- else {
1642
- a_i16_vec = _mm512_loadu_epi16(a_scalars);
1643
- b_i16_vec = _mm512_loadu_epi16(b_scalars);
1644
- a_scalars += 32, b_scalars += 32, count_scalars -= 32;
1645
- }
1646
- ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
1647
- if (count_scalars) goto simsimd_dot_f16_sapphire_cycle;
1648
-
1649
- *result = _mm512_reduce_add_ph(ab_vec);
1650
- }
1651
-
1652
- SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
1653
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1654
- __m512i a_vec, b_vec;
1655
- __m512h ab_real_vec = _mm512_setzero_ph();
1656
- __m512h ab_imag_vec = _mm512_setzero_ph();
1657
-
1658
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1659
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1660
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1661
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1662
- // one of the vectors.
1663
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1664
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1665
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1666
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1667
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1668
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1669
- );
1670
-
1671
- simsimd_dot_f16c_sapphire_cycle:
1672
- if (count_pairs < 16) {
1673
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1674
- a_vec = _mm512_maskz_loadu_epi16(mask, a_pairs);
1675
- b_vec = _mm512_maskz_loadu_epi16(mask, b_pairs);
1676
- count_pairs = 0;
1677
- }
1678
- else {
1679
- a_vec = _mm512_loadu_epi16(a_pairs);
1680
- b_vec = _mm512_loadu_epi16(b_pairs);
1681
- a_pairs += 16, b_pairs += 16, count_pairs -= 16;
1682
- }
1683
- // TODO: Consider using `_mm512_fmaddsub` and `_mm512_fcmadd_pch`
1684
- ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_xor_si512(b_vec, sign_flip_vec)),
1685
- _mm512_castsi512_ph(a_vec), ab_real_vec);
1686
- ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
1687
- _mm512_castsi512_ph(a_vec), ab_imag_vec);
1688
- if (count_pairs) goto simsimd_dot_f16c_sapphire_cycle;
1689
-
1690
- // Reduce horizontal sums:
1691
- // TODO: Optimize this with tree-like reductions
1692
- results[0] = _mm512_reduce_add_ph(ab_real_vec);
1693
- results[1] = _mm512_reduce_add_ph(ab_imag_vec);
1694
- }
1695
-
1696
- SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16c_t const *a_pairs, simsimd_f16c_t const *b_pairs,
1697
- simsimd_size_t count_pairs, simsimd_distance_t *results) {
1698
- __m512i a_vec, b_vec;
1699
- __m512h ab_real_vec = _mm512_setzero_ph();
1700
- __m512h ab_imag_vec = _mm512_setzero_ph();
1701
-
1702
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1703
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1704
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1705
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1706
- // one of the vectors.
1707
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1708
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1709
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1710
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1711
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1712
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1713
- );
1714
-
1715
- simsimd_dot_f16c_sapphire_cycle:
1716
- if (count_pairs < 16) {
1717
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_pairs * 2);
1718
- a_vec = _mm512_maskz_loadu_epi16(mask, a_pairs);
1719
- b_vec = _mm512_maskz_loadu_epi16(mask, b_pairs);
1720
- count_pairs = 0;
1721
- }
1722
- else {
1723
- a_vec = _mm512_loadu_epi16(a_pairs);
1724
- b_vec = _mm512_loadu_epi16(b_pairs);
1725
- a_pairs += 16, b_pairs += 16, count_pairs -= 16;
1726
- }
1727
- // TODO: Consider using `_mm512_fmaddsub` and `_mm512_fcmadd_pch`
1728
- ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_real_vec);
1729
- a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
1730
- b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
1731
- ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_imag_vec);
1732
- if (count_pairs) goto simsimd_dot_f16c_sapphire_cycle;
1733
-
1734
- // Reduce horizontal sums:
1735
- results[0] = _mm512_reduce_add_ph(ab_real_vec);
1736
- results[1] = _mm512_reduce_add_ph(ab_imag_vec);
1737
- }
1738
-
1739
- #pragma clang attribute pop
1740
- #pragma GCC pop_options
1741
- #endif // SIMSIMD_TARGET_SAPPHIRE
1742
-
1743
- #if SIMSIMD_TARGET_ICE
1744
- #pragma GCC push_options
1745
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1746
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1747
- apply_to = function)
1748
-
1749
- SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const *a_scalars, simsimd_i8_t const *b_scalars,
1750
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1751
- __m512i a_i16_vec, b_i16_vec;
1752
- __m512i ab_i32_vec = _mm512_setzero_si512();
1753
-
1754
- simsimd_dot_i8_ice_cycle:
1755
- if (count_scalars < 32) {
1756
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
1757
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a_scalars));
1758
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b_scalars));
1759
- count_scalars = 0;
1760
- }
1761
- else {
1762
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a_scalars));
1763
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b_scalars));
1764
- a_scalars += 32, b_scalars += 32, count_scalars -= 32;
1765
- }
1766
- // Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
1767
- // as it's asymmetric with respect to the sign of the input arguments:
1768
- // Signed(ZeroExtend16(a_scalars.byte[4*j]) * SignExtend16(b_scalars.byte[4*j]))
1769
- // So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
1770
- // to 16-bit beforehand.
1771
- ab_i32_vec = _mm512_dpwssd_epi32(ab_i32_vec, a_i16_vec, b_i16_vec);
1772
- if (count_scalars) goto simsimd_dot_i8_ice_cycle;
1773
-
1774
- *result = _mm512_reduce_add_epi32(ab_i32_vec);
1775
- }
1776
-
1777
- SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const *a_scalars, simsimd_u8_t const *b_scalars,
1778
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1779
- __m512i a_u8_vec, b_u8_vec;
1780
- __m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
1781
- __m512i ab_i32_low_vec = _mm512_setzero_si512();
1782
- __m512i ab_i32_high_vec = _mm512_setzero_si512();
1783
- __m512i const zeros_vec = _mm512_setzero_si512();
1784
-
1785
- simsimd_dot_u8_ice_cycle:
1786
- if (count_scalars < 64) {
1787
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, count_scalars);
1788
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a_scalars);
1789
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b_scalars);
1790
- count_scalars = 0;
1791
- }
1792
- else {
1793
- a_u8_vec = _mm512_loadu_si512(a_scalars);
1794
- b_u8_vec = _mm512_loadu_si512(b_scalars);
1795
- a_scalars += 64, b_scalars += 64, count_scalars -= 64;
1796
- }
1797
-
1798
- // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1799
- // instructions instead of extracts, as they are much faster and more efficient.
1800
- a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
1801
- a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
1802
- b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
1803
- b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
1804
- // Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
1805
- // as it's asymmetric with respect to the sign of the input arguments:
1806
- // Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
1807
- // So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
1808
- // to 16-bit beforehand.
1809
- ab_i32_low_vec = _mm512_dpwssd_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
1810
- ab_i32_high_vec = _mm512_dpwssd_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
1811
- if (count_scalars) goto simsimd_dot_u8_ice_cycle;
1812
-
1813
- *result = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1814
- }
1815
-
1816
- #pragma clang attribute pop
1817
- #pragma GCC pop_options
1818
- #endif // SIMSIMD_TARGET_ICE
1819
-
1820
- #if SIMSIMD_TARGET_SIERRA
1821
- #pragma GCC push_options
1822
- #pragma GCC target("avx2", "bmi2", "avx2vnni")
1823
- #pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
1824
-
1825
- SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const *a_scalars, simsimd_i8_t const *b_scalars,
1826
- simsimd_size_t count_scalars, simsimd_distance_t *result) {
1827
-
1828
- __m256i ab_i32_vec = _mm256_setzero_si256();
1829
- simsimd_size_t idx_scalars = 0;
1830
- for (; idx_scalars + 32 <= count_scalars; idx_scalars += 32) {
1831
- __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a_scalars + idx_scalars));
1832
- __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b_scalars + idx_scalars));
1833
- ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
1834
- }
1835
-
1836
- // Further reduce to a single sum for each vector
1837
- int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
1838
-
1839
- // Take care of the tail:
1840
- for (; idx_scalars < count_scalars; ++idx_scalars) ab += (int)(a_scalars[idx_scalars]) * b_scalars[idx_scalars];
1841
- *result = ab;
1842
- }
1843
-
1844
- #pragma clang attribute pop
1845
- #pragma GCC pop_options
1846
- #endif // SIMSIMD_TARGET_SIERRA
1847
- #endif // _SIMSIMD_TARGET_X86
1848
-
1849
- #ifdef __cplusplus
1850
- }
1851
- #endif
1852
-
1853
- #endif