usearch 2.21.4 → 2.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (602) hide show
  1. package/README.md +24 -24
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +7 -7
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +83 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/dist/cjs/usearch.d.ts +8 -3
  16. package/javascript/dist/cjs/usearch.js +12 -4
  17. package/javascript/dist/esm/usearch.d.ts +8 -3
  18. package/javascript/dist/esm/usearch.js +12 -4
  19. package/javascript/lib.cpp +17 -0
  20. package/javascript/usearch.ts +14 -6
  21. package/numkong/.clang-tidy +30 -0
  22. package/{simsimd → numkong}/.cmake-format.py +18 -0
  23. package/numkong/.editorconfig +12 -0
  24. package/numkong/.git-blame-ignore-revs +123 -0
  25. package/numkong/.mailmap +15 -0
  26. package/numkong/.spi.yml +11 -0
  27. package/numkong/.swift-format +14 -0
  28. package/numkong/CITATION.cff +81 -0
  29. package/numkong/CONTRIBUTING.md +500 -0
  30. package/numkong/Cargo.toml +54 -0
  31. package/numkong/MANIFEST.in +8 -0
  32. package/numkong/Package.swift +84 -0
  33. package/numkong/README.md +569 -0
  34. package/numkong/bench/README.md +217 -0
  35. package/numkong/bench/bench-browser.html +323 -0
  36. package/numkong/bench/bench.cpp +344 -0
  37. package/numkong/bench/bench.hpp +816 -0
  38. package/numkong/bench/bench.js +148 -0
  39. package/numkong/bench/bench.mjs +616 -0
  40. package/numkong/bench/bench_attention.cpp +113 -0
  41. package/numkong/bench/bench_cast.cpp +264 -0
  42. package/numkong/bench/bench_cross_amx.cpp +62 -0
  43. package/numkong/bench/bench_cross_arm.cpp +245 -0
  44. package/numkong/bench/bench_cross_blas.cpp +309 -0
  45. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  46. package/numkong/bench/bench_cross_power.cpp +84 -0
  47. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  48. package/numkong/bench/bench_cross_serial.cpp +151 -0
  49. package/numkong/bench/bench_cross_sme.cpp +163 -0
  50. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  51. package/numkong/bench/bench_cross_x86.cpp +455 -0
  52. package/numkong/bench/bench_curved.cpp +210 -0
  53. package/numkong/bench/bench_dot.cpp +315 -0
  54. package/numkong/bench/bench_each.cpp +503 -0
  55. package/numkong/bench/bench_geospatial.cpp +112 -0
  56. package/numkong/bench/bench_maxsim.cpp +154 -0
  57. package/numkong/bench/bench_mesh.cpp +148 -0
  58. package/numkong/bench/bench_probability.cpp +60 -0
  59. package/numkong/bench/bench_reduce.cpp +371 -0
  60. package/numkong/bench/bench_set.cpp +73 -0
  61. package/numkong/bench/bench_sparse.cpp +198 -0
  62. package/numkong/bench/bench_spatial.cpp +347 -0
  63. package/numkong/bench/bench_trigonometry.cpp +137 -0
  64. package/numkong/bench/swift/Bench.swift +506 -0
  65. package/numkong/binding.gyp +169 -0
  66. package/numkong/build.rs +484 -0
  67. package/numkong/c/dispatch.h +512 -0
  68. package/numkong/c/dispatch_bf16.c +447 -0
  69. package/numkong/c/dispatch_bf16c.c +52 -0
  70. package/numkong/c/dispatch_e2m3.c +296 -0
  71. package/numkong/c/dispatch_e3m2.c +284 -0
  72. package/numkong/c/dispatch_e4m3.c +318 -0
  73. package/numkong/c/dispatch_e5m2.c +316 -0
  74. package/numkong/c/dispatch_f16.c +452 -0
  75. package/numkong/c/dispatch_f16c.c +58 -0
  76. package/numkong/c/dispatch_f32.c +434 -0
  77. package/numkong/c/dispatch_f32c.c +99 -0
  78. package/numkong/c/dispatch_f64.c +348 -0
  79. package/numkong/c/dispatch_f64c.c +98 -0
  80. package/numkong/c/dispatch_i16.c +96 -0
  81. package/numkong/c/dispatch_i32.c +89 -0
  82. package/numkong/c/dispatch_i4.c +153 -0
  83. package/numkong/c/dispatch_i64.c +86 -0
  84. package/numkong/c/dispatch_i8.c +348 -0
  85. package/numkong/c/dispatch_other.c +348 -0
  86. package/numkong/c/dispatch_u1.c +193 -0
  87. package/numkong/c/dispatch_u16.c +124 -0
  88. package/numkong/c/dispatch_u32.c +118 -0
  89. package/numkong/c/dispatch_u4.c +153 -0
  90. package/numkong/c/dispatch_u64.c +102 -0
  91. package/numkong/c/dispatch_u8.c +364 -0
  92. package/numkong/c/numkong.c +940 -0
  93. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  94. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  95. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  96. package/numkong/cmake/nk_math_lib.cmake +46 -0
  97. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  99. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  100. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  101. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  102. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  103. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  104. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  105. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  106. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  107. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  108. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  109. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  110. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  111. package/numkong/include/README.md +643 -0
  112. package/numkong/include/module.modulemap +129 -0
  113. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  114. package/numkong/include/numkong/attention/sme.h +2075 -0
  115. package/numkong/include/numkong/attention.h +49 -0
  116. package/numkong/include/numkong/capabilities.h +930 -0
  117. package/numkong/include/numkong/cast/README.md +262 -0
  118. package/numkong/include/numkong/cast/diamond.h +64 -0
  119. package/numkong/include/numkong/cast/haswell.h +823 -0
  120. package/numkong/include/numkong/cast/icelake.h +475 -0
  121. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  122. package/numkong/include/numkong/cast/neon.h +1174 -0
  123. package/numkong/include/numkong/cast/powervsx.h +449 -0
  124. package/numkong/include/numkong/cast/rvv.h +970 -0
  125. package/numkong/include/numkong/cast/sapphire.h +262 -0
  126. package/numkong/include/numkong/cast/serial.h +2336 -0
  127. package/numkong/include/numkong/cast/skylake.h +915 -0
  128. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  129. package/numkong/include/numkong/cast.h +254 -0
  130. package/numkong/include/numkong/cast.hpp +44 -0
  131. package/numkong/include/numkong/curved/README.md +223 -0
  132. package/numkong/include/numkong/curved/genoa.h +182 -0
  133. package/numkong/include/numkong/curved/haswell.h +276 -0
  134. package/numkong/include/numkong/curved/neon.h +329 -0
  135. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  136. package/numkong/include/numkong/curved/rvv.h +305 -0
  137. package/numkong/include/numkong/curved/serial.h +207 -0
  138. package/numkong/include/numkong/curved/skylake.h +457 -0
  139. package/numkong/include/numkong/curved/smef64.h +530 -0
  140. package/numkong/include/numkong/curved.h +513 -0
  141. package/numkong/include/numkong/curved.hpp +144 -0
  142. package/numkong/include/numkong/dot/README.md +442 -0
  143. package/numkong/include/numkong/dot/alder.h +563 -0
  144. package/numkong/include/numkong/dot/diamond.h +158 -0
  145. package/numkong/include/numkong/dot/genoa.h +289 -0
  146. package/numkong/include/numkong/dot/haswell.h +1723 -0
  147. package/numkong/include/numkong/dot/icelake.h +997 -0
  148. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  149. package/numkong/include/numkong/dot/neon.h +869 -0
  150. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  151. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  152. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  153. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  154. package/numkong/include/numkong/dot/powervsx.h +752 -0
  155. package/numkong/include/numkong/dot/rvv.h +722 -0
  156. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  157. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  158. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  159. package/numkong/include/numkong/dot/sapphire.h +141 -0
  160. package/numkong/include/numkong/dot/serial.h +874 -0
  161. package/numkong/include/numkong/dot/sierra.h +405 -0
  162. package/numkong/include/numkong/dot/skylake.h +1116 -0
  163. package/numkong/include/numkong/dot/sve.h +420 -0
  164. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  165. package/numkong/include/numkong/dot/svehalf.h +168 -0
  166. package/numkong/include/numkong/dot/svesdot.h +90 -0
  167. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  168. package/numkong/include/numkong/dot.h +1136 -0
  169. package/numkong/include/numkong/dot.hpp +94 -0
  170. package/numkong/include/numkong/dots/README.md +496 -0
  171. package/numkong/include/numkong/dots/alder.h +117 -0
  172. package/numkong/include/numkong/dots/diamond.h +86 -0
  173. package/numkong/include/numkong/dots/genoa.h +100 -0
  174. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  175. package/numkong/include/numkong/dots/haswell.h +310 -0
  176. package/numkong/include/numkong/dots/icelake.h +180 -0
  177. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  178. package/numkong/include/numkong/dots/neon.h +123 -0
  179. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  180. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  181. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  182. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  183. package/numkong/include/numkong/dots/powervsx.h +194 -0
  184. package/numkong/include/numkong/dots/rvv.h +2593 -0
  185. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  186. package/numkong/include/numkong/dots/serial.h +2867 -0
  187. package/numkong/include/numkong/dots/sierra.h +100 -0
  188. package/numkong/include/numkong/dots/skylake.h +207 -0
  189. package/numkong/include/numkong/dots/sme.h +5041 -0
  190. package/numkong/include/numkong/dots/smebi32.h +477 -0
  191. package/numkong/include/numkong/dots/smef64.h +1328 -0
  192. package/numkong/include/numkong/dots/smehalf.h +47 -0
  193. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  194. package/numkong/include/numkong/dots.h +3110 -0
  195. package/numkong/include/numkong/dots.hpp +641 -0
  196. package/numkong/include/numkong/each/README.md +465 -0
  197. package/numkong/include/numkong/each/haswell.h +1658 -0
  198. package/numkong/include/numkong/each/icelake.h +272 -0
  199. package/numkong/include/numkong/each/neon.h +1138 -0
  200. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  201. package/numkong/include/numkong/each/neonhalf.h +318 -0
  202. package/numkong/include/numkong/each/rvv.h +1121 -0
  203. package/numkong/include/numkong/each/sapphire.h +343 -0
  204. package/numkong/include/numkong/each/serial.h +282 -0
  205. package/numkong/include/numkong/each/skylake.h +1562 -0
  206. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  207. package/numkong/include/numkong/each.h +2232 -0
  208. package/numkong/include/numkong/each.hpp +434 -0
  209. package/numkong/include/numkong/geospatial/README.md +147 -0
  210. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  211. package/numkong/include/numkong/geospatial/neon.h +615 -0
  212. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  213. package/numkong/include/numkong/geospatial/serial.h +309 -0
  214. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  215. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  216. package/numkong/include/numkong/geospatial.h +453 -0
  217. package/numkong/include/numkong/geospatial.hpp +235 -0
  218. package/numkong/include/numkong/matrix.hpp +336 -0
  219. package/numkong/include/numkong/maxsim/README.md +187 -0
  220. package/numkong/include/numkong/maxsim/alder.h +511 -0
  221. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  222. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  223. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  224. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  225. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  226. package/numkong/include/numkong/maxsim/serial.h +490 -0
  227. package/numkong/include/numkong/maxsim/sme.h +993 -0
  228. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  229. package/numkong/include/numkong/maxsim.h +571 -0
  230. package/numkong/include/numkong/maxsim.hpp +133 -0
  231. package/numkong/include/numkong/mesh/README.md +219 -0
  232. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  233. package/numkong/include/numkong/mesh/neon.h +1821 -0
  234. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  235. package/numkong/include/numkong/mesh/rvv.h +936 -0
  236. package/numkong/include/numkong/mesh/serial.h +721 -0
  237. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  238. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  239. package/numkong/include/numkong/mesh.h +666 -0
  240. package/numkong/include/numkong/mesh.hpp +701 -0
  241. package/numkong/include/numkong/numkong.h +78 -0
  242. package/numkong/include/numkong/numkong.hpp +58 -0
  243. package/numkong/include/numkong/probability/README.md +177 -0
  244. package/numkong/include/numkong/probability/haswell.h +267 -0
  245. package/numkong/include/numkong/probability/neon.h +255 -0
  246. package/numkong/include/numkong/probability/rvv.h +414 -0
  247. package/numkong/include/numkong/probability/serial.h +181 -0
  248. package/numkong/include/numkong/probability/skylake.h +324 -0
  249. package/numkong/include/numkong/probability.h +380 -0
  250. package/numkong/include/numkong/probability.hpp +120 -0
  251. package/numkong/include/numkong/random.h +50 -0
  252. package/numkong/include/numkong/random.hpp +285 -0
  253. package/numkong/include/numkong/reduce/README.md +552 -0
  254. package/numkong/include/numkong/reduce/alder.h +636 -0
  255. package/numkong/include/numkong/reduce/genoa.h +201 -0
  256. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  257. package/numkong/include/numkong/reduce/icelake.h +549 -0
  258. package/numkong/include/numkong/reduce/neon.h +3980 -0
  259. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  260. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  261. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  262. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  263. package/numkong/include/numkong/reduce/serial.h +753 -0
  264. package/numkong/include/numkong/reduce/sierra.h +338 -0
  265. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  266. package/numkong/include/numkong/reduce/sve.h +52 -0
  267. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  268. package/numkong/include/numkong/reduce.h +1582 -0
  269. package/numkong/include/numkong/reduce.hpp +778 -0
  270. package/numkong/include/numkong/scalar/README.md +89 -0
  271. package/numkong/include/numkong/scalar/haswell.h +122 -0
  272. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  273. package/numkong/include/numkong/scalar/neon.h +122 -0
  274. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  275. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  276. package/numkong/include/numkong/scalar/rvv.h +211 -0
  277. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  278. package/numkong/include/numkong/scalar/serial.h +332 -0
  279. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  280. package/numkong/include/numkong/scalar.h +696 -0
  281. package/numkong/include/numkong/set/README.md +179 -0
  282. package/numkong/include/numkong/set/haswell.h +334 -0
  283. package/numkong/include/numkong/set/icelake.h +485 -0
  284. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  285. package/numkong/include/numkong/set/neon.h +363 -0
  286. package/numkong/include/numkong/set/powervsx.h +326 -0
  287. package/numkong/include/numkong/set/rvv.h +226 -0
  288. package/numkong/include/numkong/set/rvvbb.h +117 -0
  289. package/numkong/include/numkong/set/serial.h +174 -0
  290. package/numkong/include/numkong/set/sve.h +187 -0
  291. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  292. package/numkong/include/numkong/set.h +471 -0
  293. package/numkong/include/numkong/set.hpp +114 -0
  294. package/numkong/include/numkong/sets/README.md +149 -0
  295. package/numkong/include/numkong/sets/haswell.h +63 -0
  296. package/numkong/include/numkong/sets/icelake.h +66 -0
  297. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  298. package/numkong/include/numkong/sets/neon.h +61 -0
  299. package/numkong/include/numkong/sets/powervsx.h +65 -0
  300. package/numkong/include/numkong/sets/serial.h +43 -0
  301. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  302. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  303. package/numkong/include/numkong/sets.h +382 -0
  304. package/numkong/include/numkong/sparse/README.md +156 -0
  305. package/numkong/include/numkong/sparse/icelake.h +463 -0
  306. package/numkong/include/numkong/sparse/neon.h +288 -0
  307. package/numkong/include/numkong/sparse/serial.h +117 -0
  308. package/numkong/include/numkong/sparse/sve2.h +504 -0
  309. package/numkong/include/numkong/sparse/turin.h +322 -0
  310. package/numkong/include/numkong/sparse.h +363 -0
  311. package/numkong/include/numkong/sparse.hpp +113 -0
  312. package/numkong/include/numkong/spatial/README.md +439 -0
  313. package/numkong/include/numkong/spatial/alder.h +604 -0
  314. package/numkong/include/numkong/spatial/diamond.h +240 -0
  315. package/numkong/include/numkong/spatial/genoa.h +222 -0
  316. package/numkong/include/numkong/spatial/haswell.h +979 -0
  317. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  318. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  319. package/numkong/include/numkong/spatial/neon.h +846 -0
  320. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  321. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  322. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  323. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  324. package/numkong/include/numkong/spatial/rvv.h +991 -0
  325. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  326. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  327. package/numkong/include/numkong/spatial/serial.h +347 -0
  328. package/numkong/include/numkong/spatial/sierra.h +516 -0
  329. package/numkong/include/numkong/spatial/skylake.h +605 -0
  330. package/numkong/include/numkong/spatial/sve.h +246 -0
  331. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  332. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  333. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  334. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  335. package/numkong/include/numkong/spatial.h +1681 -0
  336. package/numkong/include/numkong/spatial.hpp +183 -0
  337. package/numkong/include/numkong/spatials/README.md +580 -0
  338. package/numkong/include/numkong/spatials/alder.h +94 -0
  339. package/numkong/include/numkong/spatials/diamond.h +82 -0
  340. package/numkong/include/numkong/spatials/genoa.h +94 -0
  341. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  342. package/numkong/include/numkong/spatials/haswell.h +219 -0
  343. package/numkong/include/numkong/spatials/icelake.h +113 -0
  344. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  345. package/numkong/include/numkong/spatials/neon.h +109 -0
  346. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  347. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  348. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  349. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  350. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  351. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  352. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  353. package/numkong/include/numkong/spatials/serial.h +248 -0
  354. package/numkong/include/numkong/spatials/sierra.h +96 -0
  355. package/numkong/include/numkong/spatials/skylake.h +184 -0
  356. package/numkong/include/numkong/spatials/sme.h +1927 -0
  357. package/numkong/include/numkong/spatials/smef64.h +483 -0
  358. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  359. package/numkong/include/numkong/spatials.h +3374 -0
  360. package/numkong/include/numkong/spatials.hpp +509 -0
  361. package/numkong/include/numkong/tensor.hpp +1981 -0
  362. package/numkong/include/numkong/trigonometry/README.md +184 -0
  363. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  364. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  365. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  366. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  367. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  368. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  369. package/numkong/include/numkong/trigonometry.h +467 -0
  370. package/numkong/include/numkong/trigonometry.hpp +166 -0
  371. package/numkong/include/numkong/types.h +1695 -0
  372. package/numkong/include/numkong/types.hpp +6027 -0
  373. package/numkong/include/numkong/vector.hpp +766 -0
  374. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  375. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  376. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  377. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  378. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  380. package/numkong/javascript/README.md +254 -0
  381. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  382. package/numkong/javascript/numkong-browser.ts +40 -0
  383. package/numkong/javascript/numkong-wasm.ts +750 -0
  384. package/numkong/javascript/numkong.c +712 -0
  385. package/numkong/javascript/numkong.ts +613 -0
  386. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  387. package/numkong/javascript/tsconfig-cjs.json +13 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  389. package/numkong/javascript/types.ts +784 -0
  390. package/numkong/numkong.gypi +104 -0
  391. package/numkong/package.json +108 -0
  392. package/numkong/probes/arm_neon.c +8 -0
  393. package/numkong/probes/arm_neon_bfdot.c +9 -0
  394. package/numkong/probes/arm_neon_fhm.c +9 -0
  395. package/numkong/probes/arm_neon_half.c +8 -0
  396. package/numkong/probes/arm_neon_sdot.c +9 -0
  397. package/numkong/probes/arm_neonfp8.c +9 -0
  398. package/numkong/probes/arm_sme.c +16 -0
  399. package/numkong/probes/arm_sme2.c +16 -0
  400. package/numkong/probes/arm_sme2p1.c +16 -0
  401. package/numkong/probes/arm_sme_bf16.c +16 -0
  402. package/numkong/probes/arm_sme_bi32.c +16 -0
  403. package/numkong/probes/arm_sme_f64.c +16 -0
  404. package/numkong/probes/arm_sme_fa64.c +14 -0
  405. package/numkong/probes/arm_sme_half.c +16 -0
  406. package/numkong/probes/arm_sme_lut2.c +15 -0
  407. package/numkong/probes/arm_sve.c +18 -0
  408. package/numkong/probes/arm_sve2.c +20 -0
  409. package/numkong/probes/arm_sve2p1.c +18 -0
  410. package/numkong/probes/arm_sve_bfdot.c +20 -0
  411. package/numkong/probes/arm_sve_half.c +18 -0
  412. package/numkong/probes/arm_sve_sdot.c +21 -0
  413. package/numkong/probes/loongarch_lasx.c +12 -0
  414. package/numkong/probes/power_vsx.c +12 -0
  415. package/numkong/probes/probe.js +127 -0
  416. package/numkong/probes/riscv_rvv.c +14 -0
  417. package/numkong/probes/riscv_rvv_bb.c +15 -0
  418. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  419. package/numkong/probes/riscv_rvv_half.c +14 -0
  420. package/numkong/probes/wasm_v128relaxed.c +11 -0
  421. package/numkong/probes/x86_alder.c +17 -0
  422. package/numkong/probes/x86_diamond.c +17 -0
  423. package/numkong/probes/x86_genoa.c +17 -0
  424. package/numkong/probes/x86_graniteamx.c +19 -0
  425. package/numkong/probes/x86_haswell.c +11 -0
  426. package/numkong/probes/x86_icelake.c +17 -0
  427. package/numkong/probes/x86_sapphire.c +16 -0
  428. package/numkong/probes/x86_sapphireamx.c +18 -0
  429. package/numkong/probes/x86_sierra.c +17 -0
  430. package/numkong/probes/x86_skylake.c +15 -0
  431. package/numkong/probes/x86_turin.c +17 -0
  432. package/numkong/rust/README.md +698 -0
  433. package/numkong/rust/capabilities.rs +96 -0
  434. package/numkong/rust/cast.rs +314 -0
  435. package/numkong/rust/curved.rs +391 -0
  436. package/numkong/rust/each.rs +2721 -0
  437. package/numkong/rust/geospatial.rs +343 -0
  438. package/numkong/rust/matrix.rs +5424 -0
  439. package/numkong/rust/maxsim.rs +540 -0
  440. package/numkong/rust/mesh.rs +837 -0
  441. package/numkong/rust/numkong.rs +409 -0
  442. package/numkong/rust/probability.rs +272 -0
  443. package/numkong/rust/reduce.rs +1249 -0
  444. package/numkong/rust/set.rs +194 -0
  445. package/numkong/rust/sparse.rs +513 -0
  446. package/numkong/rust/spatial.rs +1815 -0
  447. package/numkong/rust/tensor.rs +7565 -0
  448. package/numkong/rust/types.rs +3666 -0
  449. package/numkong/rust/vector.rs +1899 -0
  450. package/numkong/rustfmt.toml +2 -0
  451. package/numkong/swift/Matrix.swift +1388 -0
  452. package/numkong/swift/MaxSim.swift +151 -0
  453. package/numkong/swift/NumKong.swift +283 -0
  454. package/numkong/swift/Numerics.swift +329 -0
  455. package/numkong/swift/README.md +433 -0
  456. package/numkong/swift/Scalar.swift +741 -0
  457. package/numkong/swift/Tensor.swift +209 -0
  458. package/numkong/test/README.md +315 -0
  459. package/numkong/test/playwright.config.ts +16 -0
  460. package/numkong/test/swift/Test.swift +534 -0
  461. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  462. package/numkong/test/test-browser-bundle.html +188 -0
  463. package/numkong/test/test-browser-runner.mjs +43 -0
  464. package/numkong/test/test-browser.html +305 -0
  465. package/numkong/test/test-dtypes.mjs +453 -0
  466. package/numkong/test/test-wasm.mjs +267 -0
  467. package/numkong/test/test.cpp +442 -0
  468. package/numkong/test/test.hpp +580 -0
  469. package/numkong/test/test.mjs +363 -0
  470. package/numkong/test/test_base.py +1041 -0
  471. package/numkong/test/test_cast.cpp +242 -0
  472. package/numkong/test/test_cross.hpp +551 -0
  473. package/numkong/test/test_cross.py +467 -0
  474. package/numkong/test/test_cross_amx.cpp +50 -0
  475. package/numkong/test/test_cross_arm.cpp +225 -0
  476. package/numkong/test/test_cross_blas.cpp +354 -0
  477. package/numkong/test/test_cross_loongarch.cpp +90 -0
  478. package/numkong/test/test_cross_power.cpp +73 -0
  479. package/numkong/test/test_cross_rvv.cpp +109 -0
  480. package/numkong/test/test_cross_serial.cpp +181 -0
  481. package/numkong/test/test_cross_sme.cpp +148 -0
  482. package/numkong/test/test_cross_wasm.cpp +141 -0
  483. package/numkong/test/test_cross_x86.cpp +442 -0
  484. package/numkong/test/test_curved.cpp +182 -0
  485. package/numkong/test/test_curved.py +207 -0
  486. package/numkong/test/test_dot.cpp +353 -0
  487. package/numkong/test/test_dot.py +296 -0
  488. package/numkong/test/test_each.cpp +437 -0
  489. package/numkong/test/test_each.py +756 -0
  490. package/numkong/test/test_geospatial.cpp +155 -0
  491. package/numkong/test/test_geospatial.py +241 -0
  492. package/numkong/test/test_maxsim.cpp +138 -0
  493. package/numkong/test/test_maxsim.py +209 -0
  494. package/numkong/test/test_mesh.cpp +203 -0
  495. package/numkong/test/test_mesh.py +394 -0
  496. package/numkong/test/test_probability.cpp +136 -0
  497. package/numkong/test/test_probability.py +166 -0
  498. package/numkong/test/test_reduce.cpp +563 -0
  499. package/numkong/test/test_reduce.py +781 -0
  500. package/numkong/test/test_set.cpp +151 -0
  501. package/numkong/test/test_set.py +120 -0
  502. package/numkong/test/test_similarities.py +579 -0
  503. package/numkong/test/test_sparse.cpp +121 -0
  504. package/numkong/test/test_sparse.py +118 -0
  505. package/numkong/test/test_spatial.cpp +518 -0
  506. package/numkong/test/test_spatial.py +269 -0
  507. package/numkong/test/test_tensor.cpp +884 -0
  508. package/numkong/test/test_tensor.py +1417 -0
  509. package/numkong/test/test_trigonometry.cpp +156 -0
  510. package/numkong/test/test_trigonometry.py +180 -0
  511. package/package.json +7 -8
  512. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  513. package/prebuilds/linux-arm64/usearch.node +0 -0
  514. package/prebuilds/linux-x64/usearch.node +0 -0
  515. package/fp16/CMakeLists.txt +0 -176
  516. package/fp16/LICENSE +0 -11
  517. package/fp16/README.md +0 -20
  518. package/fp16/bench/alt-element.cc +0 -89
  519. package/fp16/bench/from-alt-array.cc +0 -175
  520. package/fp16/bench/from-ieee-array.cc +0 -399
  521. package/fp16/bench/ieee-element.cc +0 -272
  522. package/fp16/bench/to-alt-array.cc +0 -91
  523. package/fp16/bench/to-ieee-array.cc +0 -303
  524. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  525. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  526. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  527. package/fp16/configure.py +0 -65
  528. package/fp16/confu.yaml +0 -8
  529. package/fp16/include/fp16/__init__.py +0 -0
  530. package/fp16/include/fp16/avx.py +0 -50
  531. package/fp16/include/fp16/avx2.py +0 -53
  532. package/fp16/include/fp16/bitcasts.h +0 -92
  533. package/fp16/include/fp16/fp16.h +0 -451
  534. package/fp16/include/fp16/psimd.h +0 -131
  535. package/fp16/include/fp16.h +0 -11
  536. package/fp16/jni/Android.mk +0 -6
  537. package/fp16/jni/Application.mk +0 -4
  538. package/fp16/test/alt-from-fp32-value.cc +0 -498
  539. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  540. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  541. package/fp16/test/alt-to-fp32-value.cc +0 -348
  542. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  543. package/fp16/test/bitcasts.cc +0 -56
  544. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  545. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  546. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  547. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  548. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  549. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  550. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  551. package/fp16/test/peachpy/stubs.py +0 -38
  552. package/fp16/test/tables.cc +0 -530
  553. package/fp16/test/tables.h +0 -13
  554. package/fp16/third-party/THHalf.h +0 -92
  555. package/fp16/third-party/eigen-half.h +0 -170
  556. package/fp16/third-party/float16-compressor.h +0 -80
  557. package/fp16/third-party/half.hpp +0 -2916
  558. package/fp16/third-party/npy-halffloat.h +0 -328
  559. package/simsimd/.git-blame-ignore-revs +0 -2
  560. package/simsimd/CONTRIBUTING.md +0 -321
  561. package/simsimd/Cargo.toml +0 -49
  562. package/simsimd/MANIFEST.in +0 -4
  563. package/simsimd/Package.swift +0 -28
  564. package/simsimd/README.md +0 -1493
  565. package/simsimd/binding.gyp +0 -35
  566. package/simsimd/build.rs +0 -70
  567. package/simsimd/c/lib.c +0 -374
  568. package/simsimd/include/module.modulemap +0 -4
  569. package/simsimd/include/simsimd/binary.h +0 -486
  570. package/simsimd/include/simsimd/curved.h +0 -1541
  571. package/simsimd/include/simsimd/dot.h +0 -1853
  572. package/simsimd/include/simsimd/elementwise.h +0 -2465
  573. package/simsimd/include/simsimd/geospatial.h +0 -43
  574. package/simsimd/include/simsimd/mesh.h +0 -69
  575. package/simsimd/include/simsimd/probability.h +0 -606
  576. package/simsimd/include/simsimd/simsimd.h +0 -2567
  577. package/simsimd/include/simsimd/sparse.h +0 -1384
  578. package/simsimd/include/simsimd/spatial.h +0 -2335
  579. package/simsimd/include/simsimd/types.h +0 -666
  580. package/simsimd/javascript/fallback.ts +0 -233
  581. package/simsimd/javascript/lib.c +0 -125
  582. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  583. package/simsimd/javascript/simsimd.ts +0 -179
  584. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  585. package/simsimd/package-lock.json +0 -1761
  586. package/simsimd/package.json +0 -61
  587. package/simsimd/rust/lib.rs +0 -1821
  588. package/simsimd/scripts/bench.cxx +0 -1170
  589. package/simsimd/scripts/bench.ipynb +0 -187
  590. package/simsimd/scripts/bench.js +0 -127
  591. package/simsimd/scripts/bench.rs +0 -876
  592. package/simsimd/scripts/bench_vectors.py +0 -734
  593. package/simsimd/scripts/bench_vectors_live.py +0 -213
  594. package/simsimd/scripts/test.c +0 -187
  595. package/simsimd/scripts/test.mjs +0 -220
  596. package/simsimd/scripts/test.py +0 -1505
  597. package/simsimd/swift/SimSIMD.swift +0 -149
  598. package/simsimd/swift/Test.swift +0 -128
  599. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  600. /package/{simsimd → numkong}/LICENSE +0 -0
  601. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  602. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -1,2335 +0,0 @@
1
- /**
2
- * @file spatial.h
3
- * @brief SIMD-accelerated Spatial Similarity Measures.
4
- * @author Ash Vardanian
5
- * @date March 14, 2023
6
- *
7
- * Contains:
8
- * - L2 (Euclidean) regular and squared distance
9
- * - Cosine (Angular) distance - @b not similarity!
10
- *
11
- * For datatypes:
12
- * - 64-bit IEEE floating point numbers
13
- * - 32-bit IEEE floating point numbers
14
- * - 16-bit IEEE floating point numbers
15
- * - 16-bit brain floating point numbers
16
- * - 8-bit unsigned integral numbers
17
- * - 8-bit signed integral numbers
18
- * - 4-bit signed integral numbers
19
- *
20
- * For hardware architectures:
21
- * - Arm: NEON, SVE
22
- * - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire
23
- *
24
- * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
25
- * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
26
- */
27
- #ifndef SIMSIMD_SPATIAL_H
28
- #define SIMSIMD_SPATIAL_H
29
-
30
- #include "types.h"
31
-
32
- #include "dot.h" // `_simsimd_reduce_f32x8_haswell`
33
-
34
- #ifdef __cplusplus
35
- extern "C" {
36
- #endif
37
-
38
- // clang-format off
39
-
40
- /* Serial backends for all numeric types.
41
- * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
42
- * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
43
- */
44
- SIMSIMD_PUBLIC void simsimd_l2_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
45
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
46
- SIMSIMD_PUBLIC void simsimd_cos_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
47
- SIMSIMD_PUBLIC void simsimd_l2_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
48
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
49
- SIMSIMD_PUBLIC void simsimd_cos_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
50
- SIMSIMD_PUBLIC void simsimd_l2_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
51
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
52
- SIMSIMD_PUBLIC void simsimd_cos_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
53
- SIMSIMD_PUBLIC void simsimd_l2_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
54
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
55
- SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
56
- SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
57
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
58
- SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
59
- SIMSIMD_PUBLIC void simsimd_l2_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
60
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
61
- SIMSIMD_PUBLIC void simsimd_cos_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
62
-
63
- /* Double-precision serial backends for all numeric types.
64
- * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
65
- */
66
- SIMSIMD_PUBLIC void simsimd_l2_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
67
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
68
- SIMSIMD_PUBLIC void simsimd_cos_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
69
- SIMSIMD_PUBLIC void simsimd_l2_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
70
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
71
- SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
72
- SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
73
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
74
-
75
- /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
76
- * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
77
- * server CPUs produced before 2023.
78
- */
79
- SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
80
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
81
- SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
82
- SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
83
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
84
- SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
85
- SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
86
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
87
- SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
88
- SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
89
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
90
- SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
91
- SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
92
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
93
- SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
94
- SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
95
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
96
- SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
97
-
98
- /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
99
- * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
100
- */
101
- SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
102
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
103
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
104
- SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
105
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
106
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
107
- SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
108
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
109
- SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
110
- SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
111
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
112
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
113
-
114
- /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
115
- * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
116
- * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
117
- * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
118
- * properly vectorized by recent compilers.
119
- */
120
- SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
121
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
122
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
123
- SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
124
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
125
- SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
126
- SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
127
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
128
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
129
- SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
130
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
131
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
132
- SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
133
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
134
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
135
- SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
136
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
137
- SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
138
-
139
- /* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
140
- * Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
141
- * as well as masked operations. This is enough to supersede auto-vectorization on `f32` and `f64` types.
142
- *
143
- * Sadly, we can't effectively interleave different kinds of arithmetic instructions to utilize more ports:
144
- *
145
- * > Like Intel server architectures since Skylake-X, SPR cores feature two 512-bit FMA units, and organize them in a similar fashion.
146
- * > One 512-bit FMA unit is created by fusing two 256-bit ones on port 0 and port 1. The other is added to port 5, as a server-specific
147
- * > core extension. The FMA units on port 0 and 1 are configured into 2×256-bit or 1×512-bit mode depending on whether 512-bit FMA
148
- * > instructions are present in the scheduler. That means a mix of 256-bit and 512-bit FMA instructions will not achieve higher IPC
149
- * > than executing 512-bit instructions alone.
150
- *
151
- * Source: https://chipsandcheese.com/p/a-peek-at-sapphire-rapids
152
- */
153
- SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
154
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
155
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
156
- SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
157
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
158
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
159
-
160
- /* SIMD-powered backends for AVX512 CPUs of Ice Lake generation and newer, using mixed arithmetic over 512-bit words.
161
- * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
162
- * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
163
- */
164
- SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
165
- SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
166
- SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
167
- SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
168
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
169
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
170
- SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
171
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
172
- SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
173
- SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
174
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
175
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
176
- SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
177
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
178
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
179
-
180
- /* SIMD-powered backends for AVX-INT8-VNNI extensions on Xeon 6 CPUs, including Sierra Forest and Granite Rapids.
181
- * The packs many "efficiency" cores into a single socket, avoiding heavy 512-bit operations, and focusing on 256-bit ones.
182
- */
183
- SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
184
- // clang-format on
185
-
186
- #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
187
- SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const *a, \
188
- simsimd_##input_type##_t const *b, simsimd_size_t n, \
189
- simsimd_distance_t *result) { \
190
- simsimd_##accumulator_type##_t d2 = 0; \
191
- for (simsimd_size_t i = 0; i != n; ++i) { \
192
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
193
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
194
- d2 += (ai - bi) * (ai - bi); \
195
- } \
196
- *result = d2; \
197
- }
198
-
199
- #define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
200
- SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const *a, \
201
- simsimd_##input_type##_t const *b, simsimd_size_t n, \
202
- simsimd_distance_t *result) { \
203
- simsimd_l2sq_##input_type##_##name(a, b, n, result); \
204
- *result = SIMSIMD_SQRT(*result); \
205
- }
206
-
207
- #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
208
- SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const *a, \
209
- simsimd_##input_type##_t const *b, simsimd_size_t n, \
210
- simsimd_distance_t *result) { \
211
- simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
212
- for (simsimd_size_t i = 0; i != n; ++i) { \
213
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
214
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
215
- ab += ai * bi; \
216
- a2 += ai * ai; \
217
- b2 += bi * bi; \
218
- } \
219
- if (a2 == 0 && b2 == 0) { *result = 0; } \
220
- else if (ab == 0) { *result = 1; } \
221
- else { \
222
- simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
223
- *result = unclipped_result > 0 ? unclipped_result : 0; \
224
- } \
225
- }
226
-
227
- SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
228
- SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f64_serial
229
- SIMSIMD_MAKE_L2(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f64_serial
230
-
231
- SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_serial
232
- SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_serial
233
- SIMSIMD_MAKE_L2(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_serial
234
-
235
- SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_serial
236
- SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_serial
237
- SIMSIMD_MAKE_L2(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_serial
238
-
239
- SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_serial
240
- SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_serial
241
- SIMSIMD_MAKE_L2(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_serial
242
-
243
- SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
244
- SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
245
- SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
246
-
247
- SIMSIMD_MAKE_COS(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_u8_serial
248
- SIMSIMD_MAKE_L2SQ(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_u8_serial
249
- SIMSIMD_MAKE_L2(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_u8_serial
250
-
251
- SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
252
- SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
253
- SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
254
-
255
- SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_accurate
256
- SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_accurate
257
- SIMSIMD_MAKE_L2(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_accurate
258
-
259
- SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_accurate
260
- SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
261
- SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
262
-
263
- #if _SIMSIMD_TARGET_ARM
264
- #if SIMSIMD_TARGET_NEON
265
- #pragma GCC push_options
266
- #pragma GCC target("arch=armv8-a+simd")
267
- #pragma clang attribute push(__attribute__((target("arch=armv8-a+simd"))), apply_to = function)
268
-
269
- SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_neon(simsimd_f32_t x) {
270
- return vget_lane_f32(vsqrt_f32(vdup_n_f32(x)), 0);
271
- }
272
- SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
273
- return vget_lane_f64(vsqrt_f64(vdup_n_f64(x)), 0);
274
- }
275
- SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
276
- simsimd_f32_t b2) {
277
- if (a2 == 0 && b2 == 0) return 0;
278
- if (ab == 0) return 1;
279
- simsimd_f32_t squares_arr[2] = {a2, b2};
280
- float32x2_t squares = vld1_f32(squares_arr);
281
- // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
282
- // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
283
- // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
284
- // https://github.com/lighttransport/embree-aarch64/issues/24
285
- // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
286
- float32x2_t rsqrts = vrsqrte_f32(squares);
287
- // Perform two rounds of Newton-Raphson refinement:
288
- // https://en.wikipedia.org/wiki/Newton%27s_method
289
- rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
290
- rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
291
- vst1_f32(squares_arr, rsqrts);
292
- simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
293
- return result > 0 ? result : 0;
294
- }
295
-
296
- SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
297
- simsimd_f64_t b2) {
298
- if (a2 == 0 && b2 == 0) return 0;
299
- if (ab == 0) return 1;
300
- simsimd_f64_t squares_arr[2] = {a2, b2};
301
- float64x2_t squares = vld1q_f64(squares_arr);
302
-
303
- // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
304
- // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
305
- // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
306
- // https://github.com/lighttransport/embree-aarch64/issues/24
307
- // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
308
- float64x2_t rsqrts = vrsqrteq_f64(squares);
309
- // Perform two rounds of Newton-Raphson refinement:
310
- // https://en.wikipedia.org/wiki/Newton%27s_method
311
- rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
312
- rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
313
- vst1q_f64(squares_arr, rsqrts);
314
- simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
315
- return result > 0 ? result : 0;
316
- }
317
-
318
- SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
319
- simsimd_distance_t *result) {
320
- simsimd_l2sq_f32_neon(a, b, n, result);
321
- *result = _simsimd_sqrt_f64_neon(*result);
322
- }
323
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
324
- simsimd_distance_t *result) {
325
- float32x4_t sum_vec = vdupq_n_f32(0);
326
- simsimd_size_t i = 0;
327
- for (; i + 4 <= n; i += 4) {
328
- float32x4_t a_vec = vld1q_f32(a + i);
329
- float32x4_t b_vec = vld1q_f32(b + i);
330
- float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
331
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
332
- }
333
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
334
- for (; i < n; ++i) {
335
- simsimd_f32_t diff = a[i] - b[i];
336
- sum += diff * diff;
337
- }
338
- *result = sum;
339
- }
340
-
341
- SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
342
- simsimd_distance_t *result) {
343
- float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
344
- simsimd_size_t i = 0;
345
- for (; i + 4 <= n; i += 4) {
346
- float32x4_t a_vec = vld1q_f32(a + i);
347
- float32x4_t b_vec = vld1q_f32(b + i);
348
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
349
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
350
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
351
- }
352
- simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
353
- for (; i < n; ++i) {
354
- simsimd_f32_t ai = a[i], bi = b[i];
355
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
356
- }
357
-
358
- *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
359
- }
360
-
361
- SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
362
- simsimd_distance_t *result) {
363
- simsimd_l2sq_f64_neon(a, b, n, result);
364
- *result = _simsimd_sqrt_f64_neon(*result);
365
- }
366
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
367
- simsimd_distance_t *result) {
368
- float64x2_t sum_vec = vdupq_n_f64(0);
369
- simsimd_size_t i = 0;
370
- for (; i + 2 <= n; i += 2) {
371
- float64x2_t a_vec = vld1q_f64(a + i);
372
- float64x2_t b_vec = vld1q_f64(b + i);
373
- float64x2_t diff_vec = vsubq_f64(a_vec, b_vec);
374
- sum_vec = vfmaq_f64(sum_vec, diff_vec, diff_vec);
375
- }
376
- simsimd_f64_t sum = vaddvq_f64(sum_vec);
377
- for (; i < n; ++i) {
378
- simsimd_f64_t diff = a[i] - b[i];
379
- sum += diff * diff;
380
- }
381
- *result = sum;
382
- }
383
-
384
- SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
385
- simsimd_distance_t *result) {
386
- float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
387
- simsimd_size_t i = 0;
388
- for (; i + 2 <= n; i += 2) {
389
- float64x2_t a_vec = vld1q_f64(a + i);
390
- float64x2_t b_vec = vld1q_f64(b + i);
391
- ab_vec = vfmaq_f64(ab_vec, a_vec, b_vec);
392
- a2_vec = vfmaq_f64(a2_vec, a_vec, a_vec);
393
- b2_vec = vfmaq_f64(b2_vec, b_vec, b_vec);
394
- }
395
- simsimd_f64_t ab = vaddvq_f64(ab_vec), a2 = vaddvq_f64(a2_vec), b2 = vaddvq_f64(b2_vec);
396
- for (; i < n; ++i) {
397
- simsimd_f64_t ai = a[i], bi = b[i];
398
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
399
- }
400
-
401
- *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
402
- }
403
-
404
- #pragma clang attribute pop
405
- #pragma GCC pop_options
406
- #endif // SIMSIMD_TARGET_NEON
407
-
408
- #if SIMSIMD_TARGET_NEON_F16
409
- #pragma GCC push_options
410
- #pragma GCC target("arch=armv8.2-a+simd+fp16")
411
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
412
-
413
- SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
414
- simsimd_distance_t *result) {
415
- simsimd_l2sq_f16_neon(a, b, n, result);
416
- *result = _simsimd_sqrt_f32_neon(*result);
417
- }
418
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
419
- simsimd_distance_t *result) {
420
- float32x4_t a_vec, b_vec;
421
- float32x4_t sum_vec = vdupq_n_f32(0);
422
-
423
- simsimd_l2sq_f16_neon_cycle:
424
- if (n < 4) {
425
- a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
426
- b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
427
- n = 0;
428
- }
429
- else {
430
- a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
431
- b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
432
- n -= 4, a += 4, b += 4;
433
- }
434
- float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
435
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
436
- if (n) goto simsimd_l2sq_f16_neon_cycle;
437
-
438
- *result = vaddvq_f32(sum_vec);
439
- }
440
-
441
- SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
442
- simsimd_distance_t *result) {
443
- float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
444
- float32x4_t a_vec, b_vec;
445
-
446
- simsimd_cos_f16_neon_cycle:
447
- if (n < 4) {
448
- a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
449
- b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
450
- n = 0;
451
- }
452
- else {
453
- a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
454
- b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
455
- n -= 4, a += 4, b += 4;
456
- }
457
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
458
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
459
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
460
- if (n) goto simsimd_cos_f16_neon_cycle;
461
-
462
- simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
463
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
464
- }
465
-
466
- #pragma clang attribute pop
467
- #pragma GCC pop_options
468
- #endif // SIMSIMD_TARGET_NEON_F16
469
-
470
- #if SIMSIMD_TARGET_NEON_BF16
471
- #pragma GCC push_options
472
- #pragma GCC target("arch=armv8.6-a+simd+bf16")
473
- #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
474
-
475
- SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
476
- simsimd_distance_t *result) {
477
-
478
- // Similar to `simsimd_cos_i8_neon`, we can use the `BFMMLA` instruction through
479
- // the `vbfmmlaq_f32` intrinsic to compute matrix products and later drop 1/4 of values.
480
- // The only difference is that `zip` isn't provided for `bf16` and we need to reinterpret back
481
- // and forth before zipping. Same as with integers, on modern Arm CPUs, this "smart"
482
- // approach is actually slower by around 25%.
483
- //
484
- // float32x4_t products_low_vec = vdupq_n_f32(0.0f);
485
- // float32x4_t products_high_vec = vdupq_n_f32(0.0f);
486
- // for (; i + 8 <= n; i += 8) {
487
- // bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
488
- // bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
489
- // int16x8_t a_vec_s16 = vreinterpretq_s16_bf16(a_vec);
490
- // int16x8_t b_vec_s16 = vreinterpretq_s16_bf16(b_vec);
491
- // int16x8x2_t y_w_vecs_s16 = vzipq_s16(a_vec_s16, b_vec_s16);
492
- // bfloat16x8_t y_vec = vreinterpretq_bf16_s16(y_w_vecs_s16.val[0]);
493
- // bfloat16x8_t w_vec = vreinterpretq_bf16_s16(y_w_vecs_s16.val[1]);
494
- // bfloat16x4_t a_low = vget_low_bf16(a_vec);
495
- // bfloat16x4_t b_low = vget_low_bf16(b_vec);
496
- // bfloat16x4_t a_high = vget_high_bf16(a_vec);
497
- // bfloat16x4_t b_high = vget_high_bf16(b_vec);
498
- // bfloat16x8_t x_vec = vcombine_bf16(a_low, b_low);
499
- // bfloat16x8_t v_vec = vcombine_bf16(a_high, b_high);
500
- // products_low_vec = vbfmmlaq_f32(products_low_vec, x_vec, y_vec);
501
- // products_high_vec = vbfmmlaq_f32(products_high_vec, v_vec, w_vec);
502
- // }
503
- // float32x4_t products_vec = vaddq_f32(products_high_vec, products_low_vec);
504
- // simsimd_f32_t a2 = products_vec[0], ab = products_vec[1], b2 = products_vec[3];
505
- //
506
- // Another way of accomplishing the same thing is to process the odd and even elements separately,
507
- // using special `vbfmlaltq_f32` and `vbfmlalbq_f32` intrinsics:
508
- //
509
- // ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
510
- // ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
511
- // a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
512
- // a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
513
- // b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
514
- // b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
515
- //
516
-
517
- float32x4_t ab_vec = vdupq_n_f32(0);
518
- float32x4_t a2_vec = vdupq_n_f32(0);
519
- float32x4_t b2_vec = vdupq_n_f32(0);
520
- bfloat16x8_t a_vec, b_vec;
521
-
522
- simsimd_cos_bf16_neon_cycle:
523
- if (n < 8) {
524
- a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
525
- b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
526
- n = 0;
527
- }
528
- else {
529
- a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
530
- b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
531
- n -= 8, a += 8, b += 8;
532
- }
533
- ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
534
- a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
535
- b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
536
- if (n) goto simsimd_cos_bf16_neon_cycle;
537
-
538
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
539
- simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
540
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
541
- }
542
-
543
- SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
544
- simsimd_distance_t *result) {
545
- simsimd_l2sq_bf16_neon(a, b, n, result);
546
- *result = _simsimd_sqrt_f64_neon(*result);
547
- }
548
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
549
- simsimd_distance_t *result) {
550
- float32x4_t diff_high_vec, diff_low_vec;
551
- float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
552
-
553
- simsimd_l2sq_bf16_neon_cycle:
554
- if (n < 8) {
555
- bfloat16x8_t a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
556
- bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
557
- diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
558
- diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
559
- n = 0;
560
- }
561
- else {
562
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
563
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
564
- diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
565
- diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
566
- n -= 8, a += 8, b += 8;
567
- }
568
- sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
569
- sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
570
- if (n) goto simsimd_l2sq_bf16_neon_cycle;
571
-
572
- *result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
573
- }
574
-
575
- #pragma clang attribute pop
576
- #pragma GCC pop_options
577
- #endif // SIMSIMD_TARGET_NEON_BF16
578
-
579
- #if SIMSIMD_TARGET_NEON_I8
580
- #pragma GCC push_options
581
- #pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
582
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
583
-
584
- SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
585
- simsimd_distance_t *result) {
586
- simsimd_l2sq_i8_neon(a, b, n, result);
587
- *result = _simsimd_sqrt_f32_neon(*result);
588
- }
589
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
590
- simsimd_distance_t *result) {
591
-
592
- // The naive approach is to upcast 8-bit signed integers into 16-bit signed integers
593
- // for subtraction, then multiply within 16-bit integers and accumulate the results
594
- // into 32-bit integers. This approach is slow on modern Arm CPUs. On Graviton 4,
595
- // that approach results in 17 GB/s of throughput, compared to 39 GB/s for `i8`
596
- // dot-products.
597
- //
598
- // Luckily we can use the `vabdq_s8` which technically returns `i8` values, but it's a
599
- // matter of reinterpret-casting! That approach boosts us to 33 GB/s of throughput.
600
- uint32x4_t d2_vec = vdupq_n_u32(0);
601
- simsimd_size_t i = 0;
602
- for (; i + 16 <= n; i += 16) {
603
- int8x16_t a_vec = vld1q_s8(a + i);
604
- int8x16_t b_vec = vld1q_s8(b + i);
605
- uint8x16_t d_vec = vreinterpretq_u8_s8(vabdq_s8(a_vec, b_vec));
606
- d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
607
- }
608
- simsimd_u32_t d2 = vaddvq_u32(d2_vec);
609
- for (; i < n; ++i) {
610
- simsimd_i32_t n = (simsimd_i32_t)a[i] - b[i];
611
- d2 += (simsimd_u32_t)(n * n);
612
- }
613
- *result = d2;
614
- }
615
-
616
- SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
617
- simsimd_distance_t *result) {
618
-
619
- simsimd_size_t i = 0;
620
-
621
- // Variant 1.
622
- // If the 128-bit `vdot_s32` intrinsic is unavailable, we can use the 64-bit `vdot_s32`.
623
- //
624
- // int32x4_t ab_vec = vdupq_n_s32(0);
625
- // int32x4_t a2_vec = vdupq_n_s32(0);
626
- // int32x4_t b2_vec = vdupq_n_s32(0);
627
- // for (simsimd_size_t i = 0; i != n; i += 8) {
628
- // int16x8_t a_vec = vmovl_s8(vld1_s8(a + i));
629
- // int16x8_t b_vec = vmovl_s8(vld1_s8(b + i));
630
- // int16x8_t ab_part_vec = vmulq_s16(a_vec, b_vec);
631
- // int16x8_t a2_part_vec = vmulq_s16(a_vec, a_vec);
632
- // int16x8_t b2_part_vec = vmulq_s16(b_vec, b_vec);
633
- // ab_vec = vaddq_s32(ab_vec, vaddq_s32(vmovl_s16(vget_high_s16(ab_part_vec)), //
634
- // vmovl_s16(vget_low_s16(ab_part_vec))));
635
- // a2_vec = vaddq_s32(a2_vec, vaddq_s32(vmovl_s16(vget_high_s16(a2_part_vec)), //
636
- // vmovl_s16(vget_low_s16(a2_part_vec))));
637
- // b2_vec = vaddq_s32(b2_vec, vaddq_s32(vmovl_s16(vget_high_s16(b2_part_vec)), //
638
- // vmovl_s16(vget_low_s16(b2_part_vec))));
639
- // }
640
- //
641
- // Variant 2.
642
- // With the 128-bit `vdotq_s32` intrinsic, we can use the following code:
643
- //
644
- // for (; i + 16 <= n; i += 16) {
645
- // int8x16_t a_vec = vld1q_s8(a + i);
646
- // int8x16_t b_vec = vld1q_s8(b + i);
647
- // ab_vec = vdotq_s32(ab_vec, a_vec, b_vec);
648
- // a2_vec = vdotq_s32(a2_vec, a_vec, a_vec);
649
- // b2_vec = vdotq_s32(b2_vec, b_vec, b_vec);
650
- // }
651
- //
652
- // Variant 3.
653
- // To use MMLA instructions, we need to reorganize the contents of the vectors.
654
- // On input we have `a_vec` and `b_vec`:
655
- //
656
- // a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]
657
- // b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]
658
- //
659
- // We will be multiplying matrices of size 2x8 and 8x2. So we need to perform a few shuffles:
660
- //
661
- // X =
662
- // a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7],
663
- // b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]
664
- // Y =
665
- // a[0], b[0],
666
- // a[1], b[1],
667
- // a[2], b[2],
668
- // a[3], b[3],
669
- // a[4], b[4],
670
- // a[5], b[5],
671
- // a[6], b[6],
672
- // a[7], b[7]
673
- //
674
- // V =
675
- // a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15],
676
- // b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]
677
- // W =
678
- // a[8], b[8],
679
- // a[9], b[9],
680
- // a[10], b[10],
681
- // a[11], b[11],
682
- // a[12], b[12],
683
- // a[13], b[13],
684
- // a[14], b[14],
685
- // a[15], b[15]
686
- //
687
- // Performing matrix multiplications we can aggregate into a matrix `products_low_vec` and `products_high_vec`:
688
- //
689
- // X * X, X * Y V * W, V * V
690
- // Y * X, Y * Y W * W, W * V
691
- //
692
- // Of those values we need only 3/4, as the (X * Y) and (Y * X) are the same.
693
- //
694
- // int32x4_t products_low_vec = vdupq_n_s32(0), products_high_vec = vdupq_n_s32(0);
695
- // int8x16_t a_low_b_low_vec, a_high_b_high_vec;
696
- // for (; i + 16 <= n; i += 16) {
697
- // int8x16_t a_vec = vld1q_s8(a + i);
698
- // int8x16_t b_vec = vld1q_s8(b + i);
699
- // int8x16x2_t y_w_vecs = vzipq_s8(a_vec, b_vec);
700
- // int8x16_t x_vec = vcombine_s8(vget_low_s8(a_vec), vget_low_s8(b_vec));
701
- // int8x16_t v_vec = vcombine_s8(vget_high_s8(a_vec), vget_high_s8(b_vec));
702
- // products_low_vec = vmmlaq_s32(products_low_vec, x_vec, y_w_vecs.val[0]);
703
- // products_high_vec = vmmlaq_s32(products_high_vec, v_vec, y_w_vecs.val[1]);
704
- // }
705
- // int32x4_t products_vec = vaddq_s32(products_high_vec, products_low_vec);
706
- // simsimd_i32_t a2 = products_vec[0];
707
- // simsimd_i32_t ab = products_vec[1];
708
- // simsimd_i32_t b2 = products_vec[3];
709
- //
710
- // That solution is elegant, but it requires the additional `+i8mm` extension and is currently slower,
711
- // at least on AWS Graviton 3.
712
- int32x4_t ab_vec = vdupq_n_s32(0);
713
- int32x4_t a2_vec = vdupq_n_s32(0);
714
- int32x4_t b2_vec = vdupq_n_s32(0);
715
- for (; i + 16 <= n; i += 16) {
716
- int8x16_t a_vec = vld1q_s8(a + i);
717
- int8x16_t b_vec = vld1q_s8(b + i);
718
- ab_vec = vdotq_s32(ab_vec, a_vec, b_vec);
719
- a2_vec = vdotq_s32(a2_vec, a_vec, a_vec);
720
- b2_vec = vdotq_s32(b2_vec, b_vec, b_vec);
721
- }
722
- simsimd_i32_t ab = vaddvq_s32(ab_vec);
723
- simsimd_i32_t a2 = vaddvq_s32(a2_vec);
724
- simsimd_i32_t b2 = vaddvq_s32(b2_vec);
725
-
726
- // Take care of the tail:
727
- for (; i < n; ++i) {
728
- simsimd_i32_t ai = a[i], bi = b[i];
729
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
730
- }
731
-
732
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
733
- }
734
-
735
- SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
736
- simsimd_distance_t *result) {
737
- simsimd_l2sq_u8_neon(a, b, n, result);
738
- *result = _simsimd_sqrt_f32_neon(*result);
739
- }
740
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
741
- simsimd_distance_t *result) {
742
- uint32x4_t d2_vec = vdupq_n_u32(0);
743
- simsimd_size_t i = 0;
744
- for (; i + 16 <= n; i += 16) {
745
- uint8x16_t a_vec = vld1q_u8(a + i);
746
- uint8x16_t b_vec = vld1q_u8(b + i);
747
- uint8x16_t d_vec = vabdq_u8(a_vec, b_vec);
748
- d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
749
- }
750
- simsimd_u32_t d2 = vaddvq_u32(d2_vec);
751
- for (; i < n; ++i) {
752
- simsimd_i32_t n = (simsimd_i32_t)a[i] - b[i];
753
- d2 += (simsimd_u32_t)(n * n);
754
- }
755
- *result = d2;
756
- }
757
-
758
- SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
759
- simsimd_distance_t *result) {
760
-
761
- simsimd_size_t i = 0;
762
- uint32x4_t ab_vec = vdupq_n_u32(0);
763
- uint32x4_t a2_vec = vdupq_n_u32(0);
764
- uint32x4_t b2_vec = vdupq_n_u32(0);
765
- for (; i + 16 <= n; i += 16) {
766
- uint8x16_t a_vec = vld1q_u8(a + i);
767
- uint8x16_t b_vec = vld1q_u8(b + i);
768
- ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
769
- a2_vec = vdotq_u32(a2_vec, a_vec, a_vec);
770
- b2_vec = vdotq_u32(b2_vec, b_vec, b_vec);
771
- }
772
- simsimd_u32_t ab = vaddvq_u32(ab_vec);
773
- simsimd_u32_t a2 = vaddvq_u32(a2_vec);
774
- simsimd_u32_t b2 = vaddvq_u32(b2_vec);
775
-
776
- // Take care of the tail:
777
- for (; i < n; ++i) {
778
- simsimd_u32_t ai = a[i], bi = b[i];
779
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
780
- }
781
-
782
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
783
- }
784
-
785
- #pragma clang attribute pop
786
- #pragma GCC pop_options
787
- #endif // SIMSIMD_TARGET_NEON_I8
788
-
789
- #if SIMSIMD_TARGET_SVE
790
- #pragma GCC push_options
791
- #pragma GCC target("arch=armv8.2-a+sve")
792
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
793
-
794
- SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
795
- simsimd_distance_t *result) {
796
- simsimd_l2sq_f32_sve(a, b, n, result);
797
- *result = _simsimd_sqrt_f64_neon(*result);
798
- }
799
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
800
- simsimd_distance_t *result) {
801
- simsimd_size_t i = 0;
802
- svfloat32_t d2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
803
- do {
804
- svbool_t pg_vec = svwhilelt_b32((unsigned int)i, (unsigned int)n);
805
- svfloat32_t a_vec = svld1_f32(pg_vec, a + i);
806
- svfloat32_t b_vec = svld1_f32(pg_vec, b + i);
807
- svfloat32_t a_minus_b_vec = svsub_f32_x(pg_vec, a_vec, b_vec);
808
- d2_vec = svmla_f32_x(pg_vec, d2_vec, a_minus_b_vec, a_minus_b_vec);
809
- i += svcntw();
810
- } while (i < n);
811
- simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_vec);
812
- *result = d2;
813
- }
814
-
815
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
816
- simsimd_distance_t *result) {
817
- simsimd_size_t i = 0;
818
- svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
819
- svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
820
- svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
821
- do {
822
- svbool_t pg_vec = svwhilelt_b32((unsigned int)i, (unsigned int)n);
823
- svfloat32_t a_vec = svld1_f32(pg_vec, a + i);
824
- svfloat32_t b_vec = svld1_f32(pg_vec, b + i);
825
- ab_vec = svmla_f32_x(pg_vec, ab_vec, a_vec, b_vec);
826
- a2_vec = svmla_f32_x(pg_vec, a2_vec, a_vec, a_vec);
827
- b2_vec = svmla_f32_x(pg_vec, b2_vec, b_vec, b_vec);
828
- i += svcntw();
829
- } while (i < n);
830
-
831
- simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
832
- simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
833
- simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
834
- *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
835
- }
836
-
837
- SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
838
- simsimd_distance_t *result) {
839
- simsimd_l2sq_f64_sve(a, b, n, result);
840
- *result = _simsimd_sqrt_f64_neon(*result);
841
- }
842
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
843
- simsimd_distance_t *result) {
844
- simsimd_size_t i = 0;
845
- svfloat64_t d2_vec = svdupq_n_f64(0.0, 0.0);
846
- do {
847
- svbool_t pg_vec = svwhilelt_b64((unsigned int)i, (unsigned int)n);
848
- svfloat64_t a_vec = svld1_f64(pg_vec, a + i);
849
- svfloat64_t b_vec = svld1_f64(pg_vec, b + i);
850
- svfloat64_t a_minus_b_vec = svsub_f64_x(pg_vec, a_vec, b_vec);
851
- d2_vec = svmla_f64_x(pg_vec, d2_vec, a_minus_b_vec, a_minus_b_vec);
852
- i += svcntd();
853
- } while (i < n);
854
- simsimd_f64_t d2 = svaddv_f64(svptrue_b32(), d2_vec);
855
- *result = d2;
856
- }
857
-
858
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
859
- simsimd_distance_t *result) {
860
- simsimd_size_t i = 0;
861
- svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
862
- svfloat64_t a2_vec = svdupq_n_f64(0.0, 0.0);
863
- svfloat64_t b2_vec = svdupq_n_f64(0.0, 0.0);
864
- do {
865
- svbool_t pg_vec = svwhilelt_b64((unsigned int)i, (unsigned int)n);
866
- svfloat64_t a_vec = svld1_f64(pg_vec, a + i);
867
- svfloat64_t b_vec = svld1_f64(pg_vec, b + i);
868
- ab_vec = svmla_f64_x(pg_vec, ab_vec, a_vec, b_vec);
869
- a2_vec = svmla_f64_x(pg_vec, a2_vec, a_vec, a_vec);
870
- b2_vec = svmla_f64_x(pg_vec, b2_vec, b_vec, b_vec);
871
- i += svcntd();
872
- } while (i < n);
873
-
874
- simsimd_f64_t ab = svaddv_f64(svptrue_b32(), ab_vec);
875
- simsimd_f64_t a2 = svaddv_f64(svptrue_b32(), a2_vec);
876
- simsimd_f64_t b2 = svaddv_f64(svptrue_b32(), b2_vec);
877
- *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
878
- }
879
-
880
- #pragma clang attribute pop
881
- #pragma GCC pop_options
882
- #endif // SIMSIMD_TARGET_SVE
883
-
884
- #if SIMSIMD_TARGET_SVE_F16
885
- #pragma GCC push_options
886
- #pragma GCC target("arch=armv8.2-a+sve+fp16")
887
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
888
-
889
- SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
890
- simsimd_distance_t *result) {
891
- simsimd_l2sq_f16_sve(a, b, n, result);
892
- *result = _simsimd_sqrt_f32_neon(*result);
893
- }
894
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
895
- simsimd_distance_t *result) {
896
- simsimd_size_t i = 0;
897
- svfloat16_t d2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
898
- simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
899
- simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
900
- do {
901
- svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
902
- svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
903
- svfloat16_t b_vec = svld1_f16(pg_vec, b + i);
904
- svfloat16_t a_minus_b_vec = svsub_f16_x(pg_vec, a_vec, b_vec);
905
- d2_vec = svmla_f16_x(pg_vec, d2_vec, a_minus_b_vec, a_minus_b_vec);
906
- i += svcnth();
907
- } while (i < n);
908
- simsimd_f16_for_arm_simd_t d2_f16 = svaddv_f16(svptrue_b16(), d2_vec);
909
- *result = d2_f16;
910
- }
911
-
912
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
913
- simsimd_distance_t *result) {
914
- simsimd_size_t i = 0;
915
- svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
916
- svfloat16_t a2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
917
- svfloat16_t b2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
918
- simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
919
- simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
920
- do {
921
- svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
922
- svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
923
- svfloat16_t b_vec = svld1_f16(pg_vec, b + i);
924
- ab_vec = svmla_f16_x(pg_vec, ab_vec, a_vec, b_vec);
925
- a2_vec = svmla_f16_x(pg_vec, a2_vec, a_vec, a_vec);
926
- b2_vec = svmla_f16_x(pg_vec, b2_vec, b_vec, b_vec);
927
- i += svcnth();
928
- } while (i < n);
929
-
930
- simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
931
- simsimd_f16_for_arm_simd_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
932
- simsimd_f16_for_arm_simd_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
933
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
934
- }
935
-
936
- #pragma clang attribute pop
937
- #pragma GCC pop_options
938
- #endif // SIMSIMD_TARGET_SVE_F16
939
-
940
- #if SIMSIMD_TARGET_SVE_BF16
941
- #pragma GCC push_options
942
- #pragma GCC target("arch=armv8.2-a+sve+bf16")
943
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
944
-
945
- SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
946
- simsimd_distance_t *result) {
947
- simsimd_l2sq_bf16_sve(a, b, n, result);
948
- *result = _simsimd_sqrt_f32_neon(*result);
949
- }
950
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
951
- simsimd_distance_t *result) {
952
- simsimd_size_t i = 0;
953
- svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
954
- svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
955
- simsimd_u16_t const *a = (simsimd_u16_t const *)(a_enum);
956
- simsimd_u16_t const *b = (simsimd_u16_t const *)(b_enum);
957
- do {
958
- svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
959
- svuint16_t a_vec = svld1_u16(pg_vec, a + i);
960
- svuint16_t b_vec = svld1_u16(pg_vec, b + i);
961
-
962
- // There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
963
- svbool_t pg_low_vec = svwhilelt_b32((unsigned int)(i), (unsigned int)n);
964
- svbool_t pg_high_vec = svwhilelt_b32((unsigned int)(i + svcnth() / 2), (unsigned int)n);
965
- svfloat32_t a_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(a_vec), 16));
966
- svfloat32_t a_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(a_vec), 16));
967
- svfloat32_t b_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(b_vec), 16));
968
- svfloat32_t b_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(b_vec), 16));
969
-
970
- svfloat32_t a_minus_b_low_vec = svsub_f32_x(pg_low_vec, a_low_vec, b_low_vec);
971
- svfloat32_t a_minus_b_high_vec = svsub_f32_x(pg_high_vec, a_high_vec, b_high_vec);
972
- d2_low_vec = svmla_f32_x(pg_vec, d2_low_vec, a_minus_b_low_vec, a_minus_b_low_vec);
973
- d2_high_vec = svmla_f32_x(pg_vec, d2_high_vec, a_minus_b_high_vec, a_minus_b_high_vec);
974
- i += svcnth();
975
- } while (i < n);
976
- simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_vec) + svaddv_f32(svptrue_b32(), d2_high_vec);
977
- *result = d2;
978
- }
979
-
980
- SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
981
- simsimd_distance_t *result) {
982
- simsimd_size_t i = 0;
983
- svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
984
- svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
985
- svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
986
- simsimd_bf16_for_arm_simd_t const *a = (simsimd_bf16_for_arm_simd_t const *)(a_enum);
987
- simsimd_bf16_for_arm_simd_t const *b = (simsimd_bf16_for_arm_simd_t const *)(b_enum);
988
- do {
989
- svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
990
- svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
991
- svbfloat16_t b_vec = svld1_bf16(pg_vec, b + i);
992
- ab_vec = svbfdot_f32(ab_vec, a_vec, b_vec);
993
- a2_vec = svbfdot_f32(a2_vec, a_vec, a_vec);
994
- b2_vec = svbfdot_f32(b2_vec, b_vec, b_vec);
995
- i += svcnth();
996
- } while (i < n);
997
-
998
- simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
999
- simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
1000
- simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
1001
- *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
1002
- }
1003
-
1004
- #pragma clang attribute pop
1005
- #pragma GCC pop_options
1006
- #endif // SIMSIMD_TARGET_SVE_BF16
1007
- #endif // _SIMSIMD_TARGET_ARM
1008
-
1009
- #if _SIMSIMD_TARGET_X86
1010
- #if SIMSIMD_TARGET_HASWELL
1011
- #pragma GCC push_options
1012
- #pragma GCC target("avx2")
1013
- #pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
1014
-
1015
- SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_haswell(simsimd_f32_t x) {
1016
- return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set_ss(x)));
1017
- }
1018
- SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_haswell(simsimd_f64_t x) {
1019
- return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set_sd(x)));
1020
- }
1021
-
1022
- SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f64_t ab, simsimd_f64_t a2,
1023
- simsimd_f64_t b2) {
1024
-
1025
- // If both vectors have magnitude 0, the distance is 0.
1026
- if (a2 == 0 && b2 == 0) return 0;
1027
- // If any one of the vectors is 0, the square root of the product is 0,
1028
- // the division is illformed, and the result is 1.
1029
- else if (ab == 0)
1030
- return 1;
1031
- // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
1032
- // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
1033
- // The latency of the native instruction is 4 cycles and it's broadly supported.
1034
- // For single-precision floats it has a maximum relative error of 1.5*2^-12.
1035
- // Higher precision isn't implemented on older CPUs. See `_simsimd_cos_normalize_f64_skylake` for that.
1036
- __m128d squares = _mm_set_pd(a2, b2);
1037
- __m128d rsqrts = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(squares)));
1038
- // Newton-Raphson iteration for reciprocal square root:
1039
- // https://en.wikipedia.org/wiki/Newton%27s_method
1040
- rsqrts = _mm_add_pd( //
1041
- _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
1042
- _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
1043
-
1044
- simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
1045
- simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
1046
- simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
1047
- return result > 0 ? result : 0;
1048
- }
1049
-
1050
- SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f32_t ab, simsimd_f32_t a2,
1051
- simsimd_f32_t b2) {
1052
-
1053
- // If both vectors have magnitude 0, the distance is 0.
1054
- if (a2 == 0.0f && b2 == 0.0f) return 0.0f;
1055
- // If any one of the vectors is 0, the square root of the product is 0,
1056
- // the division is illformed, and the result is 1.
1057
- else if (ab == 0.0f)
1058
- return 1.0f;
1059
-
1060
- // Load the squares into an __m128 register for single-precision floating-point operations
1061
- __m128 squares = _mm_set_ps(a2, b2, a2, b2); // We replicate to make use of full register
1062
-
1063
- // Compute the reciprocal square root of the squares using `_mm_rsqrt_ps` (single-precision)
1064
- __m128 rsqrts = _mm_rsqrt_ps(squares);
1065
-
1066
- // Perform one iteration of Newton-Raphson refinement to improve the precision of rsqrt:
1067
- // Formula: y' = y * (1.5 - 0.5 * x * y * y)
1068
- __m128 half = _mm_set1_ps(0.5f);
1069
- __m128 three_halves = _mm_set1_ps(1.5f);
1070
- rsqrts =
1071
- _mm_mul_ps(rsqrts, _mm_sub_ps(three_halves, _mm_mul_ps(half, _mm_mul_ps(squares, _mm_mul_ps(rsqrts, rsqrts)))));
1072
-
1073
- // Extract the reciprocal square roots of a2 and b2 from the __m128 register
1074
- simsimd_f32_t a2_reciprocal = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1075
- simsimd_f32_t b2_reciprocal = _mm_cvtss_f32(rsqrts);
1076
-
1077
- // Calculate the cosine distance: 1 - ab * a2_reciprocal * b2_reciprocal
1078
- simsimd_distance_t result = 1.0f - ab * a2_reciprocal * b2_reciprocal;
1079
- return result > 0 ? result : 0;
1080
- }
1081
-
1082
- #pragma clang attribute pop
1083
- #pragma GCC pop_options
1084
- #endif // SIMSIMD_TARGET_HASWELL
1085
- #endif // _SIMSIMD_TARGET_X86
1086
-
1087
- #if _SIMSIMD_TARGET_X86
1088
- #if SIMSIMD_TARGET_HASWELL
1089
- #pragma GCC push_options
1090
- #pragma GCC target("avx2", "f16c", "fma")
1091
- #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
1092
-
1093
- SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1094
- simsimd_distance_t *result) {
1095
- simsimd_l2sq_f16_haswell(a, b, n, result);
1096
- *result = _simsimd_sqrt_f32_haswell(*result);
1097
- }
1098
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1099
- simsimd_distance_t *result) {
1100
- __m256 a_vec, b_vec;
1101
- __m256 d2_vec = _mm256_setzero_ps();
1102
-
1103
- simsimd_l2sq_f16_haswell_cycle:
1104
- if (n < 8) {
1105
- a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1106
- b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1107
- n = 0;
1108
- }
1109
- else {
1110
- a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1111
- b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1112
- n -= 8, a += 8, b += 8;
1113
- }
1114
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1115
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1116
- if (n) goto simsimd_l2sq_f16_haswell_cycle;
1117
-
1118
- *result = _simsimd_reduce_f32x8_haswell(d2_vec);
1119
- }
1120
-
1121
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1122
- simsimd_distance_t *result) {
1123
- __m256 a_vec, b_vec;
1124
- __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
1125
-
1126
- simsimd_cos_f16_haswell_cycle:
1127
- if (n < 8) {
1128
- a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1129
- b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1130
- n = 0;
1131
- }
1132
- else {
1133
- a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1134
- b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1135
- n -= 8, a += 8, b += 8;
1136
- }
1137
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1138
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1139
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1140
- if (n) goto simsimd_cos_f16_haswell_cycle;
1141
-
1142
- simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1143
- simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1144
- simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1145
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1146
- }
1147
-
1148
- SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1149
- simsimd_distance_t *result) {
1150
- simsimd_l2sq_bf16_haswell(a, b, n, result);
1151
- *result = _simsimd_sqrt_f32_haswell(*result);
1152
- }
1153
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1154
- simsimd_distance_t *result) {
1155
- __m256 a_vec, b_vec;
1156
- __m256 d2_vec = _mm256_setzero_ps();
1157
-
1158
- simsimd_l2sq_bf16_haswell_cycle:
1159
- if (n < 8) {
1160
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1161
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1162
- n = 0;
1163
- }
1164
- else {
1165
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1166
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1167
- n -= 8, a += 8, b += 8;
1168
- }
1169
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1170
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1171
- if (n) goto simsimd_l2sq_bf16_haswell_cycle;
1172
-
1173
- *result = _simsimd_reduce_f32x8_haswell(d2_vec);
1174
- }
1175
-
1176
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1177
- simsimd_distance_t *result) {
1178
- __m256 a_vec, b_vec;
1179
- __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
1180
-
1181
- simsimd_cos_bf16_haswell_cycle:
1182
- if (n < 8) {
1183
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1184
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1185
- n = 0;
1186
- }
1187
- else {
1188
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1189
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1190
- n -= 8, a += 8, b += 8;
1191
- }
1192
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1193
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1194
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1195
- if (n) goto simsimd_cos_bf16_haswell_cycle;
1196
-
1197
- simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1198
- simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1199
- simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1200
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1201
- }
1202
-
1203
- SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1204
- simsimd_distance_t *result) {
1205
- simsimd_l2sq_i8_haswell(a, b, n, result);
1206
- *result = _simsimd_sqrt_f32_haswell(*result);
1207
- }
1208
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1209
- simsimd_distance_t *result) {
1210
-
1211
- __m256i d2_i32_low_vec = _mm256_setzero_si256();
1212
- __m256i d2_i32_high_vec = _mm256_setzero_si256();
1213
-
1214
- simsimd_size_t i = 0;
1215
- for (; i + 32 <= n; i += 32) {
1216
- __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1217
- __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1218
-
1219
- // Sign extend `i8` to `i16`
1220
- __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_i8_vec));
1221
- __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1222
- __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_i8_vec));
1223
- __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1224
-
1225
- // Subtract
1226
- // After this we will be squaring the values. The sign will be dropped
1227
- // and each difference will be in the range [0, 255].
1228
- __m256i d_i16_low_vec = _mm256_sub_epi16(a_i16_low_vec, b_i16_low_vec);
1229
- __m256i d_i16_high_vec = _mm256_sub_epi16(a_i16_high_vec, b_i16_high_vec);
1230
-
1231
- // Accumulate into `i32` vectors
1232
- d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1233
- d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
1234
- }
1235
-
1236
- // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1237
- int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
1238
-
1239
- // Take care of the tail:
1240
- for (; i < n; ++i) {
1241
- int n = (int)(a[i]) - b[i];
1242
- d2 += n * n;
1243
- }
1244
-
1245
- *result = (simsimd_f64_t)d2;
1246
- }
1247
-
1248
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1249
- simsimd_distance_t *result) {
1250
-
1251
- __m256i ab_i32_low_vec = _mm256_setzero_si256();
1252
- __m256i ab_i32_high_vec = _mm256_setzero_si256();
1253
- __m256i a2_i32_low_vec = _mm256_setzero_si256();
1254
- __m256i a2_i32_high_vec = _mm256_setzero_si256();
1255
- __m256i b2_i32_low_vec = _mm256_setzero_si256();
1256
- __m256i b2_i32_high_vec = _mm256_setzero_si256();
1257
-
1258
- // AVX2 has no instructions for 8-bit signed integer dot-products,
1259
- // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1260
- // So we need to normalize the first vector to its absolute value,
1261
- // and shift the product sign into the second vector.
1262
- //
1263
- // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1264
- // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1265
- // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1266
- //
1267
- // The problem with this approach, however, is the `-128` value in the second vector.
1268
- // Flipping it's sign will do nothing, and the result will be incorrect.
1269
- // This can easily lead to noticeable numerical errors in the final result.
1270
- simsimd_size_t i = 0;
1271
- for (; i + 32 <= n; i += 32) {
1272
- __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1273
- __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1274
-
1275
- // Unpack `int8` to `int16`
1276
- __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
1277
- __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1278
- __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
1279
- __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1280
-
1281
- // Multiply and accumulate as `int16`, accumulate products as `int32`:
1282
- ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1283
- ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1284
- a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1285
- a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1286
- b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1287
- b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
1288
- }
1289
-
1290
- // Further reduce to a single sum for each vector
1291
- int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1292
- int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1293
- int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
1294
-
1295
- // Take care of the tail:
1296
- for (; i < n; ++i) {
1297
- int ai = a[i], bi = b[i];
1298
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1299
- }
1300
-
1301
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1302
- }
1303
-
1304
- SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1305
- simsimd_distance_t *result) {
1306
- simsimd_l2sq_u8_haswell(a, b, n, result);
1307
- *result = _simsimd_sqrt_f32_haswell(*result);
1308
- }
1309
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1310
- simsimd_distance_t *result) {
1311
-
1312
- __m256i d2_i32_low_vec = _mm256_setzero_si256();
1313
- __m256i d2_i32_high_vec = _mm256_setzero_si256();
1314
- __m256i const zeros_vec = _mm256_setzero_si256();
1315
-
1316
- simsimd_size_t i = 0;
1317
- for (; i + 32 <= n; i += 32) {
1318
- __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1319
- __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1320
-
1321
- // Substracting unsigned vectors in AVX2 is done by saturating subtraction:
1322
- __m256i d_u8_vec = _mm256_or_si256(_mm256_subs_epu8(a_u8_vec, b_u8_vec), _mm256_subs_epu8(b_u8_vec, a_u8_vec));
1323
-
1324
- // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1325
- // instructions instead of extracts, as they are much faster and more efficient.
1326
- __m256i d_i16_low_vec = _mm256_unpacklo_epi8(d_u8_vec, zeros_vec);
1327
- __m256i d_i16_high_vec = _mm256_unpackhi_epi8(d_u8_vec, zeros_vec);
1328
-
1329
- // Multiply and accumulate at `int16` level, accumulate at `int32` level:
1330
- d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1331
- d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
1332
- }
1333
-
1334
- // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1335
- int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
1336
-
1337
- // Take care of the tail:
1338
- for (; i < n; ++i) {
1339
- int n = (int)(a[i]) - b[i];
1340
- d2 += n * n;
1341
- }
1342
-
1343
- *result = (simsimd_f64_t)d2;
1344
- }
1345
-
1346
- SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1347
- simsimd_distance_t *result) {
1348
-
1349
- __m256i ab_i32_low_vec = _mm256_setzero_si256();
1350
- __m256i ab_i32_high_vec = _mm256_setzero_si256();
1351
- __m256i a2_i32_low_vec = _mm256_setzero_si256();
1352
- __m256i a2_i32_high_vec = _mm256_setzero_si256();
1353
- __m256i b2_i32_low_vec = _mm256_setzero_si256();
1354
- __m256i b2_i32_high_vec = _mm256_setzero_si256();
1355
- __m256i const zeros_vec = _mm256_setzero_si256();
1356
-
1357
- // AVX2 has no instructions for 8-bit signed integer dot-products,
1358
- // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1359
- // So we need to normalize the first vector to its absolute value,
1360
- // and shift the product sign into the second vector.
1361
- //
1362
- // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1363
- // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1364
- // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1365
- //
1366
- // The problem with this approach, however, is the `-128` value in the second vector.
1367
- // Flipping it's sign will do nothing, and the result will be incorrect.
1368
- // This can easily lead to noticeable numerical errors in the final result.
1369
- simsimd_size_t i = 0;
1370
- for (; i + 32 <= n; i += 32) {
1371
- __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1372
- __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1373
-
1374
- // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1375
- // instructions instead of extracts, as they are much faster and more efficient.
1376
- __m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
1377
- __m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
1378
- __m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
1379
- __m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
1380
-
1381
- // Multiply and accumulate as `int16`, accumulate products as `int32`
1382
- ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1383
- ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1384
- a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1385
- a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1386
- b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1387
- b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
1388
- }
1389
-
1390
- // Further reduce to a single sum for each vector
1391
- int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1392
- int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1393
- int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
1394
-
1395
- // Take care of the tail:
1396
- for (; i < n; ++i) {
1397
- int ai = a[i], bi = b[i];
1398
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1399
- }
1400
-
1401
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1402
- }
1403
-
1404
- SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1405
- simsimd_distance_t *result) {
1406
- simsimd_l2sq_f32_haswell(a, b, n, result);
1407
- *result = _simsimd_sqrt_f32_haswell(*result);
1408
- }
1409
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1410
- simsimd_distance_t *result) {
1411
-
1412
- __m256 d2_vec = _mm256_setzero_ps();
1413
- simsimd_size_t i = 0;
1414
- for (; i + 8 <= n; i += 8) {
1415
- __m256 a_vec = _mm256_loadu_ps(a + i);
1416
- __m256 b_vec = _mm256_loadu_ps(b + i);
1417
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1418
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1419
- }
1420
-
1421
- simsimd_f64_t d2 = _simsimd_reduce_f32x8_haswell(d2_vec);
1422
- for (; i < n; ++i) {
1423
- float d = a[i] - b[i];
1424
- d2 += d * d;
1425
- }
1426
-
1427
- *result = d2;
1428
- }
1429
-
1430
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1431
- simsimd_distance_t *result) {
1432
-
1433
- __m256 ab_vec = _mm256_setzero_ps();
1434
- __m256 a2_vec = _mm256_setzero_ps();
1435
- __m256 b2_vec = _mm256_setzero_ps();
1436
- simsimd_size_t i = 0;
1437
- for (; i + 8 <= n; i += 8) {
1438
- __m256 a_vec = _mm256_loadu_ps(a + i);
1439
- __m256 b_vec = _mm256_loadu_ps(b + i);
1440
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1441
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1442
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1443
- }
1444
-
1445
- simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1446
- simsimd_f64_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1447
- simsimd_f64_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1448
- for (; i < n; ++i) {
1449
- float ai = a[i], bi = b[i];
1450
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1451
- }
1452
- *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1453
- }
1454
-
1455
- SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1456
- simsimd_distance_t *result) {
1457
- simsimd_l2sq_f64_haswell(a, b, n, result);
1458
- *result = _simsimd_sqrt_f64_haswell(*result);
1459
- }
1460
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1461
- simsimd_distance_t *result) {
1462
-
1463
- __m256d d2_vec = _mm256_setzero_pd();
1464
- simsimd_size_t i = 0;
1465
- for (; i + 4 <= n; i += 4) {
1466
- __m256d a_vec = _mm256_loadu_pd(a + i);
1467
- __m256d b_vec = _mm256_loadu_pd(b + i);
1468
- __m256d d_vec = _mm256_sub_pd(a_vec, b_vec);
1469
- d2_vec = _mm256_fmadd_pd(d_vec, d_vec, d2_vec);
1470
- }
1471
-
1472
- simsimd_f64_t d2 = _simsimd_reduce_f64x4_haswell(d2_vec);
1473
- for (; i < n; ++i) {
1474
- simsimd_f64_t d = a[i] - b[i];
1475
- d2 += d * d;
1476
- }
1477
-
1478
- *result = d2;
1479
- }
1480
-
1481
- SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1482
- simsimd_distance_t *result) {
1483
-
1484
- __m256d ab_vec = _mm256_setzero_pd();
1485
- __m256d a2_vec = _mm256_setzero_pd();
1486
- __m256d b2_vec = _mm256_setzero_pd();
1487
- simsimd_size_t i = 0;
1488
- for (; i + 4 <= n; i += 4) {
1489
- __m256d a_vec = _mm256_loadu_pd(a + i);
1490
- __m256d b_vec = _mm256_loadu_pd(b + i);
1491
- ab_vec = _mm256_fmadd_pd(a_vec, b_vec, ab_vec);
1492
- a2_vec = _mm256_fmadd_pd(a_vec, a_vec, a2_vec);
1493
- b2_vec = _mm256_fmadd_pd(b_vec, b_vec, b2_vec);
1494
- }
1495
-
1496
- simsimd_f64_t ab = _simsimd_reduce_f64x4_haswell(ab_vec);
1497
- simsimd_f64_t a2 = _simsimd_reduce_f64x4_haswell(a2_vec);
1498
- simsimd_f64_t b2 = _simsimd_reduce_f64x4_haswell(b2_vec);
1499
- for (; i < n; ++i) {
1500
- simsimd_f64_t ai = a[i], bi = b[i];
1501
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1502
- }
1503
- *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1504
- }
1505
-
1506
- #pragma clang attribute pop
1507
- #pragma GCC pop_options
1508
- #endif // SIMSIMD_TARGET_HASWELL
1509
-
1510
- #if SIMSIMD_TARGET_SKYLAKE
1511
- #pragma GCC push_options
1512
- #pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
1513
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
1514
-
1515
- SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1516
- simsimd_distance_t *result) {
1517
- simsimd_l2sq_f32_skylake(a, b, n, result);
1518
- *result = _simsimd_sqrt_f64_haswell(*result);
1519
- }
1520
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1521
- simsimd_distance_t *result) {
1522
- __m512 d2_vec = _mm512_setzero();
1523
- __m512 a_vec, b_vec;
1524
-
1525
- simsimd_l2sq_f32_skylake_cycle:
1526
- if (n < 16) {
1527
- __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1528
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1529
- b_vec = _mm512_maskz_loadu_ps(mask, b);
1530
- n = 0;
1531
- }
1532
- else {
1533
- a_vec = _mm512_loadu_ps(a);
1534
- b_vec = _mm512_loadu_ps(b);
1535
- a += 16, b += 16, n -= 16;
1536
- }
1537
- __m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
1538
- d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
1539
- if (n) goto simsimd_l2sq_f32_skylake_cycle;
1540
-
1541
- *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1542
- }
1543
-
1544
- SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f64_t ab, simsimd_f64_t a2,
1545
- simsimd_f64_t b2) {
1546
-
1547
- // If both vectors have magnitude 0, the distance is 0.
1548
- if (a2 == 0 && b2 == 0) return 0;
1549
- // If any one of the vectors is 0, the square root of the product is 0,
1550
- // the division is illformed, and the result is 1.
1551
- else if (ab == 0)
1552
- return 1;
1553
-
1554
- // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
1555
- // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
1556
- // The maximum relative error for this approximation is less than 2^-14, which is 6x lower than
1557
- // for single-precision floats in the `_simsimd_cos_normalize_f64_haswell` implementation.
1558
- // Mysteriously, MSVC has no `_mm_rsqrt14_pd` intrinsic, but has it's masked variants,
1559
- // so let's use `_mm_maskz_rsqrt14_pd(0xFF, ...)` instead.
1560
- __m128d squares = _mm_set_pd(a2, b2);
1561
- __m128d rsqrts = _mm_maskz_rsqrt14_pd(0xFF, squares);
1562
-
1563
- // Let's implement a single Newton-Raphson iteration to refine the result.
1564
- // This is how it affects downstream applications:
1565
- //
1566
- // +--------+------+----------+---------------------+---------------------+---------------------+
1567
- // | Metric | NDim | DType | Baseline Error | Old SimSIMD Error | New SimSIMD Error |
1568
- // +--------+------+----------+---------------------+---------------------+---------------------+
1569
- // | cosine | 1536 | bfloat16 | 1.89e-08 ± 1.59e-08 | 3.07e-07 ± 3.09e-07 | 3.53e-09 ± 2.70e-09 |
1570
- // | cosine | 1536 | float16 | 1.67e-02 ± 1.44e-02 | 2.68e-05 ± 1.95e-05 | 2.02e-05 ± 1.39e-05 |
1571
- // | cosine | 1536 | float32 | 2.21e-08 ± 1.65e-08 | 3.47e-07 ± 3.49e-07 | 3.77e-09 ± 2.84e-09 |
1572
- // | cosine | 1536 | float64 | 0.00e+00 ± 0.00e+00 | 3.80e-07 ± 4.50e-07 | 1.35e-11 ± 1.85e-11 |
1573
- // | cosine | 1536 | int8 | 0.00e+00 ± 0.00e+00 | 4.60e-04 ± 3.36e-04 | 4.20e-04 ± 4.88e-04 |
1574
- // +--------+------+----------+---------------------+---------------------+---------------------+
1575
- //
1576
- // https://en.wikipedia.org/wiki/Newton%27s_method
1577
- rsqrts = _mm_add_pd( //
1578
- _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
1579
- _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
1580
-
1581
- simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
1582
- simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
1583
- simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
1584
- return result > 0 ? result : 0;
1585
- }
1586
-
1587
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1588
- simsimd_distance_t *result) {
1589
- __m512 ab_vec = _mm512_setzero();
1590
- __m512 a2_vec = _mm512_setzero();
1591
- __m512 b2_vec = _mm512_setzero();
1592
- __m512 a_vec, b_vec;
1593
-
1594
- simsimd_cos_f32_skylake_cycle:
1595
- if (n < 16) {
1596
- __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1597
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1598
- b_vec = _mm512_maskz_loadu_ps(mask, b);
1599
- n = 0;
1600
- }
1601
- else {
1602
- a_vec = _mm512_loadu_ps(a);
1603
- b_vec = _mm512_loadu_ps(b);
1604
- a += 16, b += 16, n -= 16;
1605
- }
1606
- ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
1607
- a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
1608
- b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
1609
- if (n) goto simsimd_cos_f32_skylake_cycle;
1610
-
1611
- simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1612
- simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1613
- simsimd_f64_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1614
- *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1615
- }
1616
-
1617
- SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1618
- simsimd_distance_t *result) {
1619
- simsimd_l2sq_f64_skylake(a, b, n, result);
1620
- *result = _simsimd_sqrt_f64_haswell(*result);
1621
- }
1622
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1623
- simsimd_distance_t *result) {
1624
- __m512d d2_vec = _mm512_setzero_pd();
1625
- __m512d a_vec, b_vec;
1626
-
1627
- simsimd_l2sq_f64_skylake_cycle:
1628
- if (n < 8) {
1629
- __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1630
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1631
- b_vec = _mm512_maskz_loadu_pd(mask, b);
1632
- n = 0;
1633
- }
1634
- else {
1635
- a_vec = _mm512_loadu_pd(a);
1636
- b_vec = _mm512_loadu_pd(b);
1637
- a += 8, b += 8, n -= 8;
1638
- }
1639
- __m512d d_vec = _mm512_sub_pd(a_vec, b_vec);
1640
- d2_vec = _mm512_fmadd_pd(d_vec, d_vec, d2_vec);
1641
- if (n) goto simsimd_l2sq_f64_skylake_cycle;
1642
-
1643
- *result = _mm512_reduce_add_pd(d2_vec);
1644
- }
1645
-
1646
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1647
- simsimd_distance_t *result) {
1648
- __m512d ab_vec = _mm512_setzero_pd();
1649
- __m512d a2_vec = _mm512_setzero_pd();
1650
- __m512d b2_vec = _mm512_setzero_pd();
1651
- __m512d a_vec, b_vec;
1652
-
1653
- simsimd_cos_f64_skylake_cycle:
1654
- if (n < 8) {
1655
- __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1656
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1657
- b_vec = _mm512_maskz_loadu_pd(mask, b);
1658
- n = 0;
1659
- }
1660
- else {
1661
- a_vec = _mm512_loadu_pd(a);
1662
- b_vec = _mm512_loadu_pd(b);
1663
- a += 8, b += 8, n -= 8;
1664
- }
1665
- ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
1666
- a2_vec = _mm512_fmadd_pd(a_vec, a_vec, a2_vec);
1667
- b2_vec = _mm512_fmadd_pd(b_vec, b_vec, b2_vec);
1668
- if (n) goto simsimd_cos_f64_skylake_cycle;
1669
-
1670
- simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
1671
- simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
1672
- simsimd_f64_t b2 = _mm512_reduce_add_pd(b2_vec);
1673
- *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1674
- }
1675
-
1676
- #pragma clang attribute pop
1677
- #pragma GCC pop_options
1678
- #endif // SIMSIMD_TARGET_SKYLAKE
1679
-
1680
- #if SIMSIMD_TARGET_GENOA
1681
- #pragma GCC push_options
1682
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1683
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1684
- apply_to = function)
1685
-
1686
- SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
1687
-
1688
- union {
1689
- __m512 fvec;
1690
- __m512i ivec;
1691
- simsimd_f32_t f32[16];
1692
- simsimd_u16_t u16[32];
1693
- simsimd_bf16_t bf16[32];
1694
- } d_odd, d_even, d, a_f32_even, b_f32_even, d_f32_even, a_f32_odd, b_f32_odd, d_f32_odd, a, b;
1695
- a.ivec = a_i16;
1696
- b.ivec = b_i16;
1697
-
1698
- // There are several approaches to perform subtraction in `bf16`. The first one is:
1699
- //
1700
- // Perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
1701
- // expand it to 32-bit integers, then shift the bits by 16 to the left.
1702
- // Then subtract as floats, and shift back. During expansion, we will double the space,
1703
- // and should use separate registers for top and bottom halves.
1704
- // Some compilers don't have `_mm512_extracti32x8_epi32`, so we use `_mm512_extracti64x4_epi64`:
1705
- //
1706
- // a_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1707
- // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16)), 16));
1708
- // b_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1709
- // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16)), 16));
1710
- // a_f32_top.fvec =_mm512_castsi512_ps(
1711
- // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16, 1)), 16));
1712
- // b_f32_top.fvec =_mm512_castsi512_ps(
1713
- // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16, 1)), 16));
1714
- // d_f32_top.fvec = _mm512_sub_ps(a_f32_top.fvec, b_f32_top.fvec);
1715
- // d_f32_bot.fvec = _mm512_sub_ps(a_f32_bot.fvec, b_f32_bot.fvec);
1716
- // d.ivec = _mm512_castsi256_si512(_mm512_cvtepi32_epi16(
1717
- // _mm512_srli_epi32(_mm512_castps_si512(d_f32_bot.fvec), 16)));
1718
- // d.ivec = _mm512_inserti64x4(d.ivec, _mm512_cvtepi32_epi16(
1719
- // _mm512_srli_epi32(_mm512_castps_si512(d_f32_top.fvec), 16)), 1);
1720
- //
1721
- // Instead of using multple shifts and an insertion, we can achieve similar result with fewer expensive
1722
- // calls to `_mm512_permutex2var_epi16`, or a cheap `_mm512_mask_shuffle_epi8` and blend:
1723
- //
1724
- a_f32_odd.ivec = _mm512_and_si512(a_i16, _mm512_set1_epi32(0xFFFF0000));
1725
- a_f32_even.ivec = _mm512_slli_epi32(a_i16, 16);
1726
- b_f32_odd.ivec = _mm512_and_si512(b_i16, _mm512_set1_epi32(0xFFFF0000));
1727
- b_f32_even.ivec = _mm512_slli_epi32(b_i16, 16);
1728
-
1729
- d_f32_odd.fvec = _mm512_sub_ps(a_f32_odd.fvec, b_f32_odd.fvec);
1730
- d_f32_even.fvec = _mm512_sub_ps(a_f32_even.fvec, b_f32_even.fvec);
1731
-
1732
- d_f32_even.ivec = _mm512_srli_epi32(d_f32_even.ivec, 16);
1733
- d.ivec = _mm512_mask_blend_epi16(0x55555555, d_f32_odd.ivec, d_f32_even.ivec);
1734
-
1735
- return d.ivec;
1736
- }
1737
-
1738
- SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1739
- simsimd_distance_t *result) {
1740
- simsimd_l2sq_bf16_genoa(a, b, n, result);
1741
- *result = _simsimd_sqrt_f32_haswell(*result);
1742
- }
1743
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1744
- simsimd_distance_t *result) {
1745
- __m512 d2_vec = _mm512_setzero_ps();
1746
- __m512i a_i16_vec, b_i16_vec, d_i16_vec;
1747
-
1748
- simsimd_l2sq_bf16_genoa_cycle:
1749
- if (n < 32) {
1750
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1751
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1752
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1753
- n = 0;
1754
- }
1755
- else {
1756
- a_i16_vec = _mm512_loadu_epi16(a);
1757
- b_i16_vec = _mm512_loadu_epi16(b);
1758
- a += 32, b += 32, n -= 32;
1759
- }
1760
- d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
1761
- d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
1762
- if (n) goto simsimd_l2sq_bf16_genoa_cycle;
1763
-
1764
- *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1765
- }
1766
-
1767
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1768
- simsimd_distance_t *result) {
1769
- __m512 ab_vec = _mm512_setzero_ps();
1770
- __m512 a2_vec = _mm512_setzero_ps();
1771
- __m512 b2_vec = _mm512_setzero_ps();
1772
- __m512i a_i16_vec, b_i16_vec;
1773
-
1774
- simsimd_cos_bf16_genoa_cycle:
1775
- if (n < 32) {
1776
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1777
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1778
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1779
- n = 0;
1780
- }
1781
- else {
1782
- a_i16_vec = _mm512_loadu_epi16(a);
1783
- b_i16_vec = _mm512_loadu_epi16(b);
1784
- a += 32, b += 32, n -= 32;
1785
- }
1786
- ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
1787
- a2_vec = _mm512_dpbf16_ps(a2_vec, (__m512bh)(a_i16_vec), (__m512bh)(a_i16_vec));
1788
- b2_vec = _mm512_dpbf16_ps(b2_vec, (__m512bh)(b_i16_vec), (__m512bh)(b_i16_vec));
1789
- if (n) goto simsimd_cos_bf16_genoa_cycle;
1790
-
1791
- simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1792
- simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1793
- simsimd_f32_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1794
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1795
- }
1796
-
1797
- #pragma clang attribute pop
1798
- #pragma GCC pop_options
1799
- #endif // SIMSIMD_TARGET_GENOA
1800
-
1801
- #if SIMSIMD_TARGET_SAPPHIRE
1802
- #pragma GCC push_options
1803
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
1804
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1805
-
1806
- SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1807
- simsimd_distance_t *result) {
1808
- simsimd_l2sq_f16_sapphire(a, b, n, result);
1809
- *result = _simsimd_sqrt_f32_haswell(*result);
1810
- }
1811
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1812
- simsimd_distance_t *result) {
1813
- __m512h d2_vec = _mm512_setzero_ph();
1814
- __m512i a_i16_vec, b_i16_vec;
1815
-
1816
- simsimd_l2sq_f16_sapphire_cycle:
1817
- if (n < 32) {
1818
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1819
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1820
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1821
- n = 0;
1822
- }
1823
- else {
1824
- a_i16_vec = _mm512_loadu_epi16(a);
1825
- b_i16_vec = _mm512_loadu_epi16(b);
1826
- a += 32, b += 32, n -= 32;
1827
- }
1828
- __m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec));
1829
- d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
1830
- if (n) goto simsimd_l2sq_f16_sapphire_cycle;
1831
-
1832
- *result = _mm512_reduce_add_ph(d2_vec);
1833
- }
1834
-
1835
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1836
- simsimd_distance_t *result) {
1837
- __m512h ab_vec = _mm512_setzero_ph();
1838
- __m512h a2_vec = _mm512_setzero_ph();
1839
- __m512h b2_vec = _mm512_setzero_ph();
1840
- __m512i a_i16_vec, b_i16_vec;
1841
-
1842
- simsimd_cos_f16_sapphire_cycle:
1843
- if (n < 32) {
1844
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1845
- a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1846
- b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1847
- n = 0;
1848
- }
1849
- else {
1850
- a_i16_vec = _mm512_loadu_epi16(a);
1851
- b_i16_vec = _mm512_loadu_epi16(b);
1852
- a += 32, b += 32, n -= 32;
1853
- }
1854
- ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
1855
- a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(a_i16_vec), a2_vec);
1856
- b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_i16_vec), _mm512_castsi512_ph(b_i16_vec), b2_vec);
1857
- if (n) goto simsimd_cos_f16_sapphire_cycle;
1858
-
1859
- simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
1860
- simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
1861
- simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
1862
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1863
- }
1864
-
1865
- #pragma clang attribute pop
1866
- #pragma GCC pop_options
1867
- #endif // SIMSIMD_TARGET_SAPPHIRE
1868
-
1869
- #if SIMSIMD_TARGET_ICE
1870
- #pragma GCC push_options
1871
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1872
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1873
- apply_to = function)
1874
-
1875
- SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1876
- simsimd_distance_t *result) {
1877
- simsimd_l2sq_i8_ice(a, b, n, result);
1878
- *result = _simsimd_sqrt_f32_haswell(*result);
1879
- }
1880
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1881
- simsimd_distance_t *result) {
1882
- __m512i d2_i32_vec = _mm512_setzero_si512();
1883
- __m512i a_i16_vec, b_i16_vec, d_i16s_vec;
1884
-
1885
- simsimd_l2sq_i8_ice_cycle:
1886
- if (n < 32) { // TODO: Avoid eaarly i16 upcast to step through 64 values at a time
1887
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1888
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1889
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1890
- n = 0;
1891
- }
1892
- else {
1893
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1894
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1895
- a += 32, b += 32, n -= 32;
1896
- }
1897
- d_i16s_vec = _mm512_sub_epi16(a_i16_vec, b_i16_vec);
1898
- d2_i32_vec = _mm512_dpwssd_epi32(d2_i32_vec, d_i16s_vec, d_i16s_vec);
1899
- if (n) goto simsimd_l2sq_i8_ice_cycle;
1900
-
1901
- *result = _mm512_reduce_add_epi32(d2_i32_vec);
1902
- }
1903
-
1904
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1905
- simsimd_distance_t *result) {
1906
-
1907
- __m512i ab_i32_vec = _mm512_setzero_si512();
1908
- __m512i a2_i32_vec = _mm512_setzero_si512();
1909
- __m512i b2_i32_vec = _mm512_setzero_si512();
1910
- __m512i a_i16_vec, b_i16_vec;
1911
- simsimd_cos_i8_ice_cycle:
1912
- if (n < 32) {
1913
- __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1914
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1915
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1916
- n = 0;
1917
- }
1918
- else {
1919
- a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1920
- b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1921
- a += 32, b += 32, n -= 32;
1922
- }
1923
-
1924
- // We can't directly use the `_mm512_dpbusd_epi32` intrinsic everywhere,
1925
- // as it's asymmetric with respect to the sign of the input arguments:
1926
- //
1927
- // Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
1928
- //
1929
- // To compute the squares, we could just drop the sign bit of the second argument.
1930
- // But this would lead to big-big problems on values like `-128`!
1931
- // For dot-products we don't have the luxury of optimizing the sign bit away.
1932
- // Assuming this is an approximate kernel (with reciprocal square root approximations)
1933
- // in the end, we can allow clamping the value to [-127, 127] range.
1934
- //
1935
- // On Ice Lake:
1936
- //
1937
- // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can only execute on port 0, with 5 cycle latency.
1938
- // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also only execute on port 0, with 5 cycle latency.
1939
- // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 5, with 5 cycle latency.
1940
- //
1941
- // On Zen4 Genoa:
1942
- //
1943
- // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 4 cycle latency.
1944
- // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also execute on ports 0 and 1, with 4 cycle latency.
1945
- // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 3 cycle latency.
1946
- //
1947
- // The old solution was complex replied on 1. and 2.:
1948
- //
1949
- // a_i8_abs_vec = _mm512_abs_epi8(a_i8_vec);
1950
- // b_i8_abs_vec = _mm512_abs_epi8(b_i8_vec);
1951
- // a2_i32_vec = _mm512_dpbusds_epi32(a2_i32_vec, a_i8_abs_vec, a_i8_abs_vec);
1952
- // b2_i32_vec = _mm512_dpbusds_epi32(b2_i32_vec, b_i8_abs_vec, b_i8_abs_vec);
1953
- // ab_i32_low_vec = _mm512_dpwssds_epi32( //
1954
- // ab_i32_low_vec, //
1955
- // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)), //
1956
- // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1957
- // ab_i32_high_vec = _mm512_dpwssds_epi32( //
1958
- // ab_i32_high_vec, //
1959
- // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)), //
1960
- // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1961
- //
1962
- // The new solution is simpler and relies on 3.:
1963
- ab_i32_vec = _mm512_add_epi32(ab_i32_vec, _mm512_madd_epi16(a_i16_vec, b_i16_vec));
1964
- a2_i32_vec = _mm512_add_epi32(a2_i32_vec, _mm512_madd_epi16(a_i16_vec, a_i16_vec));
1965
- b2_i32_vec = _mm512_add_epi32(b2_i32_vec, _mm512_madd_epi16(b_i16_vec, b_i16_vec));
1966
- if (n) goto simsimd_cos_i8_ice_cycle;
1967
-
1968
- int ab = _mm512_reduce_add_epi32(ab_i32_vec);
1969
- int a2 = _mm512_reduce_add_epi32(a2_i32_vec);
1970
- int b2 = _mm512_reduce_add_epi32(b2_i32_vec);
1971
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1972
- }
1973
- SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1974
- simsimd_distance_t *result) {
1975
- simsimd_l2sq_u8_ice(a, b, n, result);
1976
- *result = _simsimd_sqrt_f32_haswell(*result);
1977
- }
1978
- SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1979
- simsimd_distance_t *result) {
1980
- __m512i d2_i32_low_vec = _mm512_setzero_si512();
1981
- __m512i d2_i32_high_vec = _mm512_setzero_si512();
1982
- __m512i const zeros_vec = _mm512_setzero_si512();
1983
- __m512i d_i16_low_vec, d_i16_high_vec;
1984
- __m512i a_u8_vec, b_u8_vec, d_u8_vec;
1985
-
1986
- simsimd_l2sq_u8_ice_cycle:
1987
- if (n < 64) {
1988
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1989
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1990
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1991
- n = 0;
1992
- }
1993
- else {
1994
- a_u8_vec = _mm512_loadu_si512(a);
1995
- b_u8_vec = _mm512_loadu_si512(b);
1996
- a += 64, b += 64, n -= 64;
1997
- }
1998
-
1999
- // Substracting unsigned vectors in AVX-512 is done by saturating subtraction:
2000
- d_u8_vec = _mm512_or_si512(_mm512_subs_epu8(a_u8_vec, b_u8_vec), _mm512_subs_epu8(b_u8_vec, a_u8_vec));
2001
- d_i16_low_vec = _mm512_unpacklo_epi8(d_u8_vec, zeros_vec);
2002
- d_i16_high_vec = _mm512_unpackhi_epi8(d_u8_vec, zeros_vec);
2003
-
2004
- // Multiply and accumulate at `int16` level, accumulate at `int32` level:
2005
- d2_i32_low_vec = _mm512_dpwssd_epi32(d2_i32_low_vec, d_i16_low_vec, d_i16_low_vec);
2006
- d2_i32_high_vec = _mm512_dpwssd_epi32(d2_i32_high_vec, d_i16_high_vec, d_i16_high_vec);
2007
- if (n) goto simsimd_l2sq_u8_ice_cycle;
2008
-
2009
- *result = _mm512_reduce_add_epi32(_mm512_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
2010
- }
2011
-
2012
- SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
2013
- simsimd_distance_t *result) {
2014
-
2015
- __m512i ab_i32_low_vec = _mm512_setzero_si512();
2016
- __m512i ab_i32_high_vec = _mm512_setzero_si512();
2017
- __m512i a2_i32_low_vec = _mm512_setzero_si512();
2018
- __m512i a2_i32_high_vec = _mm512_setzero_si512();
2019
- __m512i b2_i32_low_vec = _mm512_setzero_si512();
2020
- __m512i b2_i32_high_vec = _mm512_setzero_si512();
2021
- __m512i const zeros_vec = _mm512_setzero_si512();
2022
- __m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
2023
- __m512i a_u8_vec, b_u8_vec;
2024
-
2025
- simsimd_cos_u8_ice_cycle:
2026
- if (n < 64) {
2027
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
2028
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
2029
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
2030
- n = 0;
2031
- }
2032
- else {
2033
- a_u8_vec = _mm512_loadu_si512(a);
2034
- b_u8_vec = _mm512_loadu_si512(b);
2035
- a += 64, b += 64, n -= 64;
2036
- }
2037
-
2038
- // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
2039
- // instructions instead of extracts, as they are much faster and more efficient.
2040
- a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
2041
- a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
2042
- b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
2043
- b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
2044
-
2045
- // Multiply and accumulate as `int16`, accumulate products as `int32`:
2046
- ab_i32_low_vec = _mm512_dpwssds_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
2047
- ab_i32_high_vec = _mm512_dpwssds_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
2048
- a2_i32_low_vec = _mm512_dpwssds_epi32(a2_i32_low_vec, a_i16_low_vec, a_i16_low_vec);
2049
- a2_i32_high_vec = _mm512_dpwssds_epi32(a2_i32_high_vec, a_i16_high_vec, a_i16_high_vec);
2050
- b2_i32_low_vec = _mm512_dpwssds_epi32(b2_i32_low_vec, b_i16_low_vec, b_i16_low_vec);
2051
- b2_i32_high_vec = _mm512_dpwssds_epi32(b2_i32_high_vec, b_i16_high_vec, b_i16_high_vec);
2052
- if (n) goto simsimd_cos_u8_ice_cycle;
2053
-
2054
- int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2055
- int a2 = _mm512_reduce_add_epi32(_mm512_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
2056
- int b2 = _mm512_reduce_add_epi32(_mm512_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
2057
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2058
- }
2059
-
2060
- SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2061
- simsimd_distance_t *result) {
2062
- simsimd_l2sq_i4x2_ice(a, b, n_words, result);
2063
- *result = _simsimd_sqrt_f32_haswell(*result);
2064
- }
2065
- SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2066
- simsimd_distance_t *result) {
2067
-
2068
- // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2069
- // The absolute difference between two 4-bit integers is at most 15 and it is always a `uint4_t` value!
2070
- // Moreover, it's square is at most 225, which fits into `uint8_t` and can be computed with a single
2071
- // lookup table. Accumulating those values is similar to checksumming, a piece of cake for SIMD!
2072
- __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2073
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2074
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2075
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2076
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2077
- __m512i const u4_squares_lookup_vec = _mm512_set_epi8( //
2078
- (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2079
- (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2080
- (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2081
- (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2082
-
2083
- /// The mask used to take the low nibble of each byte.
2084
- __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2085
-
2086
- // Temporaries:
2087
- __m512i a_i4x2_vec, b_i4x2_vec;
2088
- __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2089
- __m512i d_u8_low_vec, d_u8_high_vec; //! Only the low 4 bits are actually used
2090
- __m512i d2_u8_low_vec, d2_u8_high_vec;
2091
- __m512i d2_u16_low_vec, d2_u16_high_vec;
2092
-
2093
- // Accumulators:
2094
- __m512i d2_u32_vec = _mm512_setzero_si512();
2095
-
2096
- simsimd_l2sq_i4x2_ice_cycle:
2097
- if (n_words < 64) {
2098
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2099
- a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2100
- b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2101
- n_words = 0;
2102
- }
2103
- else {
2104
- a_i4x2_vec = _mm512_loadu_epi8(a);
2105
- b_i4x2_vec = _mm512_loadu_epi8(b);
2106
- a += 64, b += 64, n_words -= 64;
2107
- }
2108
-
2109
- // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2110
- a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2111
- a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2112
- b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2113
- b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2114
- a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2115
- a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2116
- b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2117
- b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
2118
-
2119
- // We can implement subtraction with a lookup table, or using `_mm512_sub_epi8`.
2120
- d_u8_low_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_low_vec, b_i8_low_vec));
2121
- d_u8_high_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_high_vec, b_i8_high_vec));
2122
-
2123
- // Now we can use the lookup table to compute the squares of the 4-bit unsigned integers
2124
- // in the low nibbles of the `d_u8_low_vec` and `d_u8_high_vec` vectors.
2125
- d2_u8_low_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_low_vec);
2126
- d2_u8_high_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_high_vec);
2127
-
2128
- // Aggregating into 16-bit integers, we need to first upcast our 8-bit values to 16 bits.
2129
- // After that, we will perform one more operation, upcasting further into 32-bit integers.
2130
- d2_u16_low_vec = //
2131
- _mm512_add_epi16( //
2132
- _mm512_unpacklo_epi8(d2_u8_low_vec, _mm512_setzero_si512()),
2133
- _mm512_unpackhi_epi8(d2_u8_low_vec, _mm512_setzero_si512()));
2134
- d2_u16_high_vec = //
2135
- _mm512_add_epi16( //
2136
- _mm512_unpacklo_epi8(d2_u8_high_vec, _mm512_setzero_si512()),
2137
- _mm512_unpackhi_epi8(d2_u8_high_vec, _mm512_setzero_si512()));
2138
- d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_low_vec, _mm512_setzero_si512()));
2139
- d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_high_vec, _mm512_setzero_si512()));
2140
- if (n_words) goto simsimd_l2sq_i4x2_ice_cycle;
2141
-
2142
- // Finally, we can reduce the 16-bit integers to 32-bit integers and sum them up.
2143
- int d2 = _mm512_reduce_add_epi32(d2_u32_vec);
2144
- *result = d2;
2145
- }
2146
- SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2147
- simsimd_distance_t *result) {
2148
-
2149
- // We need to compose a lookup table for all the scalar products of 4-bit integers.
2150
- // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2151
- // Practically speaking, the product of two 4-bit signed integers is a 7-bit integer,
2152
- // as the maximum absolute value of the product is `abs(-8 * -8) == 64`.
2153
- //
2154
- // To store 128 possible values of 2^7 bits we only need 128 single-byte scalars,
2155
- // or just 2x ZMM registers. In that case our lookup will only take `vpermi2b` instruction,
2156
- // easily inokable with `_mm512_permutex2var_epi8` intrinsic with latency of 6 on Sapphire Rapids.
2157
- // The problem is converting 2d indices of our symmetric matrix into 1d offsets in the dense array.
2158
- //
2159
- // Alternatively, we can take the entire symmetric (16 x 16) matrix of products,
2160
- // put into 4x ZMM registers, and use it with `_mm512_shuffle_epi8`, remembering
2161
- // that it can only lookup with 128-bit lanes (16x 8-bit values).
2162
- // That intrinsic has latency 1, but will need to be repeated and combined with
2163
- // multiple iterations of `_mm512_shuffle_i64x2` that has latency 3.
2164
- //
2165
- // Altenatively, we can get down to 3 cycles per lookup with `vpermb` and `_mm512_permutexvar_epi8` intrinsics.
2166
- // For that we can split our (16 x 16) matrix into 4x (8 x 8) submatrices, and use 4x ZMM registers.
2167
- //
2168
- // Still, all of those solutions are quite heavy compared to two parallel calls to `_mm512_dpbusds_epi32`
2169
- // for the dot product. But we can still use the `_mm512_permutexvar_epi8` to compute the squares of the
2170
- // 16 possible `int4_t` values faster.
2171
- //
2172
- // Here is how our `int4_t` range looks:
2173
- //
2174
- // dec: 0 1 2 3 4 5 6 7 -8 -7 -6 -5 -4 -3 -2 -1
2175
- // hex: 0 1 2 3 4 5 6 7 8 9 A B C D E F
2176
- //
2177
- // Squared:
2178
- //
2179
- // dec2: 0 1 4 9 16 25 36 49 64 49 36 25 16 9 4 1
2180
- // hex2: 0 1 4 9 10 19 24 31 40 31 24 19 10 9 4 1
2181
- //
2182
- // Broadcast it to every lane, so that: `square(x) == _mm512_shuffle_epi8(i4_squares_lookup_vec, x)`.
2183
- __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2184
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2185
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2186
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2187
- -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2188
- __m512i const i4_squares_lookup_vec = _mm512_set_epi8( //
2189
- 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2190
- 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2191
- 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2192
- 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2193
-
2194
- /// The mask used to take the low nibble of each byte.
2195
- __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2196
-
2197
- // Temporaries:
2198
- __m512i a_i4x2_vec, b_i4x2_vec;
2199
- __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2200
- __m512i a2_u8_vec, b2_u8_vec;
2201
-
2202
- // Accumulators:
2203
- __m512i a2_u16_low_vec = _mm512_setzero_si512();
2204
- __m512i a2_u16_high_vec = _mm512_setzero_si512();
2205
- __m512i b2_u16_low_vec = _mm512_setzero_si512();
2206
- __m512i b2_u16_high_vec = _mm512_setzero_si512();
2207
- __m512i ab_i32_low_vec = _mm512_setzero_si512();
2208
- __m512i ab_i32_high_vec = _mm512_setzero_si512();
2209
-
2210
- simsimd_cos_i4x2_ice_cycle:
2211
- if (n_words < 64) {
2212
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2213
- a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2214
- b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2215
- n_words = 0;
2216
- }
2217
- else {
2218
- a_i4x2_vec = _mm512_loadu_epi8(a);
2219
- b_i4x2_vec = _mm512_loadu_epi8(b);
2220
- a += 64, b += 64, n_words -= 64;
2221
- }
2222
-
2223
- // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2224
- // For now, they are not really 8-bit integers, as they are not sign-extended.
2225
- // That part will come later, using the `i4_to_i8_lookup_vec` lookup.
2226
- a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2227
- a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2228
- b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2229
- b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2230
-
2231
- // Compute the squares of the 4-bit integers.
2232
- // For symmetry we could have used 4 registers, aka "a2_i8_low_vec", "a2_i8_high_vec", "b2_i8_low_vec",
2233
- // "b2_i8_high_vec". But the largest square value is just 64, so we can safely aggregate into 8-bit unsigned values.
2234
- a2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_low_vec),
2235
- _mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_high_vec));
2236
- b2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_low_vec),
2237
- _mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_high_vec));
2238
-
2239
- // We can safely aggregate into just 16-bit sums without overflow, if the vectors have less than:
2240
- // (2 scalars / byte) * (64 bytes / register) * (256 non-overflowing 8-bit additions in 16-bit intesgers)
2241
- // = 32'768 dimensions.
2242
- //
2243
- // We use saturated addition here to clearly inform in case of overflow.
2244
- a2_u16_low_vec = _mm512_adds_epu16(a2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2245
- a2_u16_high_vec = _mm512_adds_epu16(a2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2246
- b2_u16_low_vec = _mm512_adds_epu16(b2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2247
- b2_u16_high_vec = _mm512_adds_epu16(b2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2248
-
2249
- // Time to perform the proper sign extension of the 4-bit integers to 8-bit integers.
2250
- a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2251
- a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2252
- b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2253
- b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
2254
-
2255
- // The same trick won't work for the primary dot-product, as the signs vector
2256
- // components may differ significantly. So we have to use two `_mm512_dpwssds_epi32`
2257
- // intrinsics instead, upcasting four chunks to 16-bit integers beforehand!
2258
- // Alternatively, we can flip the signs of the second argument and use `_mm512_dpbusds_epi32`,
2259
- // but it ends up taking more instructions.
2260
- ab_i32_low_vec = _mm512_dpwssds_epi32( //
2261
- ab_i32_low_vec, //
2262
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_low_vec)), //
2263
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_low_vec)));
2264
- ab_i32_low_vec = _mm512_dpwssds_epi32( //
2265
- ab_i32_low_vec, //
2266
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_low_vec, 1)), //
2267
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_low_vec, 1)));
2268
- ab_i32_high_vec = _mm512_dpwssds_epi32( //
2269
- ab_i32_high_vec, //
2270
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_high_vec)), //
2271
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_high_vec)));
2272
- ab_i32_high_vec = _mm512_dpwssds_epi32( //
2273
- ab_i32_high_vec, //
2274
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_high_vec, 1)), //
2275
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_high_vec, 1)));
2276
- if (n_words) goto simsimd_cos_i4x2_ice_cycle;
2277
-
2278
- int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2279
- unsigned short a2_u16[32], b2_u16[32];
2280
- _mm512_storeu_si512(a2_u16, _mm512_add_epi16(a2_u16_low_vec, a2_u16_high_vec));
2281
- _mm512_storeu_si512(b2_u16, _mm512_add_epi16(b2_u16_low_vec, b2_u16_high_vec));
2282
- unsigned int a2 = 0, b2 = 0;
2283
- for (int i = 0; i < 32; ++i) a2 += a2_u16[i], b2 += b2_u16[i];
2284
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2285
- }
2286
-
2287
- #pragma clang attribute pop
2288
- #pragma GCC pop_options
2289
- #endif // SIMSIMD_TARGET_ICE
2290
-
2291
- #if SIMSIMD_TARGET_SIERRA
2292
- #pragma GCC push_options
2293
- #pragma GCC target("avx2", "bmi2", "avx2vnni")
2294
- #pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
2295
-
2296
- SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
2297
- simsimd_distance_t *result) {
2298
-
2299
- __m256i ab_i32_vec = _mm256_setzero_si256();
2300
- __m256i a2_i32_vec = _mm256_setzero_si256();
2301
- __m256i b2_i32_vec = _mm256_setzero_si256();
2302
-
2303
- simsimd_size_t i = 0;
2304
- for (; i + 32 <= n; i += 32) {
2305
- __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
2306
- __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
2307
- ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
2308
- a2_i32_vec = _mm256_dpbssds_epi32(a2_i32_vec, a_i8_vec, a_i8_vec);
2309
- b2_i32_vec = _mm256_dpbssds_epi32(b2_i32_vec, b_i8_vec, b_i8_vec);
2310
- }
2311
-
2312
- // Further reduce to a single sum for each vector
2313
- int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
2314
- int a2 = _simsimd_reduce_i32x8_haswell(a2_i32_vec);
2315
- int b2 = _simsimd_reduce_i32x8_haswell(b2_i32_vec);
2316
-
2317
- // Take care of the tail:
2318
- for (; i < n; ++i) {
2319
- int ai = a[i], bi = b[i];
2320
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
2321
- }
2322
-
2323
- *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2324
- }
2325
-
2326
- #pragma clang attribute pop
2327
- #pragma GCC pop_options
2328
- #endif // SIMSIMD_TARGET_SIERRA
2329
- #endif // _SIMSIMD_TARGET_X86
2330
-
2331
- #ifdef __cplusplus
2332
- }
2333
- #endif
2334
-
2335
- #endif